Skip to content

Commit

Permalink
Remove the rest of BUILD_PY and the old python build wiring
Browse files Browse the repository at this point in the history 
Fixes deephaven#430
  • Loading branch information
@niloc132
niloc132 committed Jun 22, 2021
1 parent 9798e27 commit ec9507c
Show file tree
Hide file tree
Showing 14 changed files with 263 additions and 1,985 deletions.
78 changes: 35 additions & 43 deletions DB-test/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,51 +35,43 @@ dependencies {
testRuntime project(':suanshu-extension')
}

if (PyEnv.pythonEnabled(project)) {
for (pv in PythonVersion.values()) {
PyInstall install = pv.getInstall(project)
Venv venvTest = install.getVenv(VenvType.TEST_DH)
venvTest.javaTest(project, "test-${pv.name}").maxHeapSize = '4G'
}
} else {
def gradleWrapper = tasks.register("dockerGradleInit", Wrapper.class) { wrapper ->
wrapper.scriptFile "${buildDir}/template-project/gradlew"
wrapper.jarFile "${buildDir}/template-project/gradle/wrapper/gradle-wrapper.jar"
}
tasks.getByName('check').dependsOn Docker.registerDockerTask(project, 'test-in-docker') {
copyIn {
dependsOn gradleWrapper
from ("${buildDir}/template-project") {
into 'project'
}
from(project.file('src/test/build.gradle.template')) {
into 'project'
rename { file -> 'build.gradle' }
}
from (sourceSets.test.runtimeClasspath) {
into 'classpath'
}
from (sourceSets.test.output.getClassesDirs()) {
into 'classes'
}
def gradleWrapper = tasks.register("dockerGradleInit", Wrapper.class) { wrapper ->
wrapper.scriptFile "${buildDir}/template-project/gradlew"
wrapper.jarFile "${buildDir}/template-project/gradle/wrapper/gradle-wrapper.jar"
}
tasks.getByName('check').dependsOn Docker.registerDockerTask(project, 'test-in-docker') {
copyIn {
dependsOn gradleWrapper
from ("${buildDir}/template-project") {
into 'project'
}
parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
dockerfile {
// base image with default java, python, wheels
from 'deephaven/runtime-base:local-build'

// set up the project
copyFile 'project', '/project'
workingDir '/project'
// run once with no actual classes, so that gradle is preinstalled and cached
runCommand '/project/gradlew'
copyFile 'classpath', '/classpath'
copyFile 'classes', '/classes'
from(project.file('src/test/build.gradle.template')) {
into 'project'
rename { file -> 'build.gradle' }
}
entrypoint = ['/project/gradlew', 'test', '--info']
containerOutPath = '/project/build/test-results/test/'
copyOut {
into "${buildDir}/test-results/test-in-docker"
from (sourceSets.test.runtimeClasspath) {
into 'classpath'
}
from (sourceSets.test.output.getClassesDirs()) {
into 'classes'
}
}
parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
dockerfile {
// base image with default java, python, wheels
from 'deephaven/runtime-base:local-build'

// set up the project
copyFile 'project', '/project'
workingDir '/project'
// run once with no actual classes, so that gradle is preinstalled and cached
runCommand '/project/gradlew'
copyFile 'classpath', '/classpath'
copyFile 'classes', '/classes'
}
entrypoint = ['/project/gradlew', 'test', '--info']
containerOutPath = '/project/build/test-results/test/'
copyOut {
into "${buildDir}/test-results/test-in-docker"
}
}
111 changes: 48 additions & 63 deletions Generators/Generators.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,79 +150,64 @@ task generatePlottingConvenienceAssert(type: JavaExec, dependsOn: 'classes') {
systemProperty 'Configuration.rootFile', 'dh-defaults.prop'
}

if (PyEnv.pythonEnabled(project)) {
PyInstall install = PythonVersion.PY_37.getInstall(project)
Venv venv = install.getVenv(VenvType.PY_DOCS)
venv.pythonExec(project, "generatePyDocAssert", ["javadocExtraction.py", devRoot, "true"], [":allJavadoc", venv.taskNameCreateVenv], "${project.projectDir}/python")
venv.pythonExec(project, "generatePyDoc", ["javadocExtraction.py", devRoot, "false"], [":allJavadoc", venv.taskNameCreateVenv], "${project.projectDir}/python")
venv.pythonExec(project, "generateManPyDocAssert", ["manualDocstrings.py", devRoot, "true"], venv.taskNameCreateVenv, "${project.projectDir}/python")
venv.pythonExec(project, "generateManPyDoc", ["manualDocstrings.py", devRoot, "false"], venv.taskNameCreateVenv, "${project.projectDir}/python")
} else {
TaskProvider<Task> generatePyDoc = Docker.registerDockerTask(project, "generatePyDoc") {
copyIn {
from('python') {
into 'python'
}

// TODO move these to a later sync task, so that we detect changes and minimally build images
// Copy existing generated JSON to where the image can get it - remove if we stop checking in JSON
from("$devRoot/Integrations/python/deephaven") {
// allow javadocExtraction.py to verify existing json
include 'doc'
// allow manualDocstrings.py to verify existing json
include 'docCustom'
into 'out'
}
// Depend on generated javadoc so we can generate JSON from it
dependsOn ':allJavadoc'
from("$devRoot/build/docs/javadoc") {
// input for javadocExtraction.py
into 'javadoc'
}
TaskProvider<Task> generatePyDoc = Docker.registerDockerTask(project, "generatePyDoc") {
copyIn {
from('python') {
into 'python'
}

dockerfile {
from 'docker.io/library/python:3.7.10'
// TODO move these to a later sync task, so that we detect changes and minimally build images
// Copy existing generated JSON to where the image can get it - remove if we stop checking in JSON
from("$devRoot/Integrations/python/deephaven") {
// allow javadocExtraction.py to verify existing json
include 'doc'
// allow manualDocstrings.py to verify existing json
include 'docCustom'
into 'out'
}
// Depend on generated javadoc so we can generate JSON from it
dependsOn ':allJavadoc'
from("$devRoot/build/docs/javadoc") {
// input for javadocExtraction.py
into 'javadoc'
}
}

runCommand 'pip3 install beautifulsoup4==4.9.3 lxml==4.6.3'
dockerfile {
from 'docker.io/library/python:3.7.10'

copyFile('.', '.')
runCommand 'pip3 install beautifulsoup4==4.9.3 lxml==4.6.3'

runCommand '''set -eux; \\
python3 /python/javadocExtraction.py / false; \\
python3 /python/manualDocstrings.py / false
'''
}
copyFile('.', '.')

parentContainers = [project(':deephaven-jpy').tasks.findByName("buildDockerForRuntime")]// deephaven/java-and-python
imageName = 'deephaven/pydocs:local-build'
copyOut {
into("$devRoot/Integrations/python/deephaven")
preserve {// only overwrite doc and docCustom directories
include '**/*'
exclude 'doc'
exclude 'docCustom'
}
}
runCommand '''set -eux; \\
python3 /python/javadocExtraction.py / false; \\
python3 /python/manualDocstrings.py / false
'''
}

tasks.register("generateManPyDoc") {
// TODO core#430 remove this task when BUILD_PY disappears
// for now, this task just makes sure that the main task ran
dependsOn generatePyDoc
}
tasks.register("generateManPyDocAssert") {
doLast {
logger.warn "This task no longer checks output, we're expecting to always regenerate now"
}
}
tasks.register("generatePyDocAssert") {
doLast {
logger.warn "This task no longer checks output, we're expecting to always regenerate now"
parentContainers = [project(':deephaven-jpy').tasks.findByName("buildDockerForRuntime")]// deephaven/java-and-python
imageName = 'deephaven/pydocs:local-build'
copyOut {
into("$devRoot/Integrations/python/deephaven")
preserve {// only overwrite doc and docCustom directories
include '**/*'
exclude 'doc'
exclude 'docCustom'
}
}
}

tasks.register("generateManPyDocAssert") {
doLast {
logger.warn "This task no longer checks output, we're expecting to always regenerate now"
}
}
tasks.register("generatePyDocAssert") {
doLast {
logger.warn "This task no longer checks output, we're expecting to always regenerate now"
}
}

def pythonStaticMethodsArgs = ['io.deephaven.util.calendar.Calendars',
'/Integrations/python/deephaven/Calendars/__init__.py',
Expand Down Expand Up @@ -258,7 +243,7 @@ task generatePythonIntegrationStaticMethods(type: JavaExec, dependsOn: 'classes'
systemProperty 'workspace', workspace
systemProperty 'devroot', devRoot
systemProperty 'Configuration.rootFile', 'dh-defaults.prop'
dependsOn ':Generators:generatePyDoc', ':Generators:generateManPyDoc'
dependsOn generatePyDoc
}

task generatePythonIntegrationStaticMethodsAssert(type: JavaExec, dependsOn: 'classes') {
Expand All @@ -285,7 +270,7 @@ task generatePythonFigureWrapper(type: JavaExec, dependsOn: 'classes') {
systemProperty 'workspace', workspace
systemProperty 'devroot', devRoot
systemProperty 'Configuration.rootFile', 'dh-defaults.prop'
dependsOn ':Generators:generatePyDoc', ':Generators:generateFigureImmutable'
dependsOn generatePyDoc, ':Generators:generateFigureImmutable'
}

task generatePythonFigureWrapperAssert(type: JavaExec, dependsOn: 'classes') {
Expand Down
136 changes: 56 additions & 80 deletions Integrations/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,89 +105,65 @@ Docker.registerDockerImage(project, 'buildDeephavenPython') {
images.add('deephaven/runtime-base:local-build')
}

/**
* We are not yet enabling the building of jpy on all environments yet.
* It requires the ability of the running machine to build/install python:
*
* https://github.com/JetBrains/gradle-python-envs
* https://github.com/pyenv/pyenv/wiki/common-build-problems
* https://github.com/pyenv/pyenv/wiki
*
* TODO IDO-436: zip up working installations on each OS, and pull them from artifactory instead of building on developer machine
*
*/
if (PyEnv.pythonEnabled(project)) {
PyEnv env = PyEnv.getEnv(project)

Exec wheel = env.getTaskBuildDhWheel()

for (pv in [PythonVersion.PY_37]) {
PyInstall install = env.getInstall(pv)
Venv venvTest = install.getVenv(VenvType.TEST_DH)
venvTest.pythonTest(project, "test-${pv.name}")
}

} else {
JavaPluginConvention java = project.convention.plugins.get('java') as JavaPluginConvention
SourceSet test = java.sourceSets.maybeCreate('test')

def runInDocker = { String name, List<String> command ->
Docker.registerDockerTask(project, name) {
copyIn {
from('python') {
into 'python'
}
from(test.runtimeClasspath) {
into 'classpath'
}

// Unpack the config contents for now, since we don't seem to read the configs from inside a jar.
// This does not add a task dependency, but we already put :configs in the testRuntime classpath,
// so it is part of the previous statement
from(zipTree(project(':configs').tasks.getByName('jar').outputs.files.singleFile)) {
into 'python/configs'
}
JavaPluginConvention java = project.convention.plugins.get('java') as JavaPluginConvention
SourceSet test = java.sourceSets.maybeCreate('test')

def runInDocker = { String name, List<String> command ->
Docker.registerDockerTask(project, name) {
copyIn {
from('python') {
into 'python'
}
parentContainers = [tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base

imageName = 'deephaven/py-integrations:local-build'
dockerfile {
// set up the container, env vars - things that aren't likely to change
from 'deephaven/runtime-base:local-build'
runCommand '''set -eux; \\
pip3 install unittest-xml-reporting==3.0.4;\\
mkdir -p /out/report;\\
mkdir -p /workspace/cache/classes'''
environmentVariable 'DEEPHAVEN_CLASSPATH', '/classpath/*:/classpath'
environmentVariable 'DEEPHAVEN_WORKSPACE', '/workspace'
environmentVariable 'DEEPHAVEN_DEVROOT', '/python'
environmentVariable 'DEEPHAVEN_PROPFILE', 'dh-defaults.prop'
environmentVariable 'JDK_HOME', '/usr/lib/jvm/zulu8/jre/'
environmentVariable 'JAVA_VERSION', '1.8'
environmentVariable 'DEEPHAVEN_VERSION', project.version

workingDir '/python'

// copy in the contents that we do expect to change as the project updates
copyFile 'python', '/python'
copyFile 'classpath', '/classpath'
from(test.runtimeClasspath) {
into 'classpath'
}
entrypoint = command

copyOut {
into layout.buildDirectory.dir('test-results')
// Unpack the config contents for now, since we don't seem to read the configs from inside a jar.
// This does not add a task dependency, but we already put :configs in the testRuntime classpath,
// so it is part of the previous statement
from(zipTree(project(':configs').tasks.getByName('jar').outputs.files.singleFile)) {
into 'python/configs'
}
}
parentContainers = [tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base

imageName = 'deephaven/py-integrations:local-build'
dockerfile {
// set up the container, env vars - things that aren't likely to change
from 'deephaven/runtime-base:local-build'
runCommand '''set -eux; \\
pip3 install unittest-xml-reporting==3.0.4;\\
mkdir -p /out/report;\\
mkdir -p /workspace/cache/classes'''
environmentVariable 'DEEPHAVEN_CLASSPATH', '/classpath/*:/classpath'
environmentVariable 'DEEPHAVEN_WORKSPACE', '/workspace'
environmentVariable 'DEEPHAVEN_DEVROOT', '/python'
environmentVariable 'DEEPHAVEN_PROPFILE', 'dh-defaults.prop'
environmentVariable 'JDK_HOME', '/usr/lib/jvm/zulu8/jre/'
environmentVariable 'JAVA_VERSION', '1.8'
environmentVariable 'DEEPHAVEN_VERSION', project.version

workingDir '/python'

// copy in the contents that we do expect to change as the project updates
copyFile 'python', '/python'
copyFile 'classpath', '/classpath'
}
entrypoint = command

copyOut {
into layout.buildDirectory.dir('test-results')
}
}
def pyTest = runInDocker("test-py-37", ['python3', '-m', 'xmlrunner', 'discover', '-v', '-o', '/out/report'])
def pyFunctionalTest = runInDocker('py-functional-test', ['/bin/bash', 'run-functional-tests.sh'])
pyTest.configure({
onlyIf { TestTools.shouldRunTests(project) }
})
tasks.getByName('test').dependsOn(pyTest)

pyFunctionalTest.configure({
onlyIf { TestTools.shouldRunTests(project) }
})
tasks.getByName('test').dependsOn(pyFunctionalTest)
}
}
def pyTest = runInDocker("test-py-37", ['python3', '-m', 'xmlrunner', 'discover', '-v', '-o', '/out/report'])
def pyFunctionalTest = runInDocker('py-functional-test', ['/bin/bash', 'run-functional-tests.sh'])
pyTest.configure({
onlyIf { TestTools.shouldRunTests(project) }
})
tasks.getByName('test').dependsOn(pyTest)

pyFunctionalTest.configure({
onlyIf { TestTools.shouldRunTests(project) }
})
tasks.getByName('test').dependsOn(pyFunctionalTest)
Loading

0 comments on commit ec9507c

Please sign in to comment.