WIP deephaven#430, with a few hacks - commit needs to be rewritten

Integrations/build.gradle

@@ -15,16 +15,11 @@ configurations {
 dependencies {
 
     compile project(':DB'), project(':DbTypes')
-
+    testRuntimeOnly project(':log-to-slf4j')
     // add configs, and some runtime dependencies to test classpaths
     testRuntime project(':configs'), project(':test-configs'),
             // need these projects so session default imports / generated type wrappers resolve correctly
             project(':Numerics'), project(':Plot')
-
-            // it's not clear why this isn't automatically hooked up; there _is_ no test java sources,
-            // but the main java source output _should_ be automatically applied.
-            sourceSets.main.output
-
 }
 
 sourceSets {
@@ -70,6 +65,44 @@ idea {
         }
 }
 
+task prepareDockerForWheel(type: Sync) {
+    from 'python'
+    into "${buildDir}/docker"
+}
+
+task buildDockerForWheel(type: DockerBuildImage) {
+    dependsOn prepareDockerForWheel
+    buildArgs.put('DEEPHAVEN_VERSION', "${project.version}")
+    images.add('deephaven/deephaven-wheel')
+    target.set('build')
+}
+
+/**
+ * Produce a docker image with the prereqs to run and use deephaven python
+ */
+task deephavenPythonDockerfile(type: Dockerfile) {
+    destFile.set layout.buildDirectory.file("$buildDir/deephaven-python-docker/Dockerfile")
+    from new Dockerfile.From('deephaven/deephaven-jpy-wheel').withStage('deephaven-jpy-wheel')
+    from new Dockerfile.From('deephaven/deephaven-wheel').withStage('deephaven-wheel')
+    from new Dockerfile.From('deephaven/java-and-python')
+
+    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven-jpy-wheel'))
+    copyFile(new Dockerfile.CopyFile('/usr/src/app/dist', '.').withStage('deephaven-wheel'))
+
+    runCommand('''set -eux;\\
+  pip3 install setuptools wheel; \\
+  pip3 install *.whl; \\
+  rm *.whl
+''')
+}
+task buildDeephavenPython(type: DockerBuildImage) {
+    //
+    inputs.files deephavenPythonDockerfile.outputs.files
+    inputDir.set layout.buildDirectory.dir("$buildDir/deephaven-python-docker")
+
+    images.add('deephaven/runtime-base')
+}
+
 /**
  * We are not yet enabling the building of jpy on all environments yet.
  * It requires the ability of the running machine to build/install python:
@@ -93,37 +126,73 @@ if (PyEnv.pythonEnabled(project)) {
     }
 
 } else {
-//    Configuration cp = project.configurations.getByName(classpathConfiguration)
-//    def pyTest = Docker.registerDockerTask(project, "test-py-37") {
-//        copyIn {
-//            from('python') {
-//                into 'python'
-//            }
-//        }
-//        parentContainers = [project(':grpc-api-base').tasks.findByName('buildDocker')] // deephaven/grpc-api-base
-//
-//        Dockerfile d = task("dockerfile", type:Dockerfile) {
-//            from "deephaven/grpc-api-base"
-//            copyFile 'python', '/python'
-//            runCommand '''set -eux;\\
-//mkdir /report; \\
-//cd /python; \\
-//python3 -m xmlrunner discover -v -o /report'''
-//        }
-//    }
-//    pyTest.configure({t ->
-//
-//    })
-}
-
-task prepareDockerForWheel(type: Sync) {
-    from 'python'
-    into "${buildDir}/docker"
-}
+    JavaPluginConvention java = project.convention.plugins.get('java') as JavaPluginConvention
+    SourceSet test = java.sourceSets.maybeCreate('test')
+
+    def pyTest = Docker.registerDockerTask(project, "test-py-37") {
+        copyIn {
+            from('python') {
+                into 'python'
+            }
+            from(test.runtimeClasspath) {
+                into 'classpath'
+            }
+
+            // Unpack the config contents for now, since we don't seem to read the configs from inside a jar.
+            // This does not add a task dependency, but we already put :configs in the testRuntime classpath,
+            // so it is part of the previous statement
+            from(zipTree(project(':configs').tasks.getByName('jar').outputs.files.singleFile)) {
+                into 'python/configs'
+            }
+        }
+        parentContainers = [tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+
+        dockerfile {
+            // 'set up the container, env vars - things that aren\'t likely to change'
+            from 'deephaven/runtime-base'
+            runCommand 'pip3 install unittest-xml-reporting==3.0.4'
+            environmentVariable 'DEEPHAVEN_CLASSPATH', '/classpath/*:/classpath'
+            environmentVariable 'DEEPHAVEN_WORKSPACE', '/workspace'
+            environmentVariable 'DEEPHAVEN_DEVROOT', '/python'
+            environmentVariable 'DEEPHAVEN_PROPFILE', 'iris-defaults.prop'
+            environmentVariable 'JDK_HOME', '/usr/lib/jvm/zulu8/jre/'
+            environmentVariable 'JAVA_VERSION', '1.8'
+            environmentVariable 'DEEPHAVEN_VERSION', project.version
+
+            // 'copy in the contents that we do expect to change as the project updates'
+            copyFile 'python', '/python'
+            copyFile 'classpath', '/classpath'
+
+            // 'run the tests and report results, status code'
+            runCommand '''set -eux;\\
+                          mkdir -p /out/report; \\
+                          mkdir -p /workspace/cache/classes; \\
+                          cd /python; \\
+                          set +e; \\
+                          python3 -m xmlrunner discover -v -o /out/report; \\
+                          echo $? > /out/exitCode'''
+        }
 
-task buildDockerForWheel(type: DockerBuildImage) {
-    dependsOn prepareDockerForWheel
-    buildArgs.put('DEEPHAVEN_VERSION', "${project.version}")
-    images.add('deephaven/deephaven-wheel')
-    target.set('build')
+        copyOut {
+            into "$buildDir/test-results"
+        }
+    }
+    pyTest.configure({
+        onlyIf { TestTools.shouldRunTests(project) }
+        doLast {
+            def exitCodeFile = new File("$buildDir/test-results/exitCode")
+            if (exitCodeFile.exists()) {
+                int exitCode = exitCodeFile.text.toInteger()
+                if (exitCode != 0) {
+                    String error = """$path failed w/ exit code $exitCode
+    See $buildDir/test-results/report/."""
+                    if (TestTools.allowFailure(gradle.rootProject)) {
+                        logger.error(error)
+                    } else {
+                        throw new GradleException(error)
+                    }
+                }
+            }
+        }
+    })
 }

grpc-api/base/build.gradle

@@ -3,25 +3,27 @@ import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 plugins {
     id 'com.bmuschko.docker-remote-api'
 }
-
-task prepareDocker(type: Sync) {
-    from(project.projectDir) {
-        exclude 'build'
-        exclude 'build.gradle'
-    }
-    into 'build/docker'
-}
-
-task buildDocker(type: DockerBuildImage) {
-    dependsOn prepareDocker
-    // deephaven/runtime
-    def jpyRuntime = project(':deephaven-jpy').tasks.findByName('buildDockerForRuntime')
-    // deephaven/deephaven-jpy-wheel
-    def jpyWheel = project(':deephaven-jpy').tasks.findByName('buildDockerForWheel')
-    // deephaven/deephaven-wheel
-    def dhWheel = project(':Integrations').tasks.findByName('buildDockerForWheel')
-    dependsOn jpyRuntime, jpyWheel, dhWheel
-    inputs.files( [jpyRuntime, jpyWheel, dhWheel].each { t -> t.outputs.files.singleFile })
-
-    images.add('deephaven/grpc-api-base')
-}
+//evaluationDependsOn ':deephaven-jpy'
+//evaluationDependsOn ':Integrations'
+//
+//task prepareDocker(type: Sync) {
+//    from(project.projectDir) {
+//        exclude 'build'
+//        exclude 'build.gradle'
+//    }
+//    into 'build/docker'
+//}
+//
+//task buildDocker(type: DockerBuildImage) {
+//    dependsOn prepareDocker
+//    // deephaven/runtime
+//    def jpyRuntime = project(':deephaven-jpy').tasks.findByName('buildDockerForRuntime')
+//    // deephaven/deephaven-jpy-wheel
+//    def jpyWheel = project(':deephaven-jpy').tasks.findByName('buildDockerForWheel')
+//    // deephaven/deephaven-wheel
+//    def dhWheel = project(':Integrations').tasks.findByName('buildDockerForWheel')
+//    dependsOn jpyRuntime, jpyWheel, dhWheel
+//    inputs.files( [jpyRuntime, jpyWheel, dhWheel].each { t -> t.outputs.files.singleFile })
+//
+//    images.add('deephaven/grpc-api-base')
+//}

grpc-api/server/docker/build.gradle

@@ -74,8 +74,8 @@ docker {
 }
 
 
-// deephaven/grpc-api-base
-def grpcApiBase = project(':grpc-api-base').tasks.findByName('buildDocker')
+// deephaven/runtime-base
+def grpcApiBase = project(':Integrations').tasks.findByName('buildDeephavenPython')
 dockerBuildImage.dependsOn grpcApiBase
 dockerBuildImage.inputs.file grpcApiBase.outputs.files.singleFile
 // Currently, GH build-ci.yml calls dockerCreateDockerfile, and relies on buildx; so we need to

grpc-api/server/docker/gradle.properties

@@ -1,5 +1,5 @@
 grpc-api-docker.imageName=deephaven/grpc-api
-grpc-api-docker.baseImage=deephaven/grpc-api-base
+grpc-api-docker.baseImage=deephaven/runtime-base
 grpc-api-docker.vendor=Deephaven Data Labs
 grpc-api-docker.title=Deephaven gRPC API
 grpc-api-docker.description=The Deephaven API - TODO

sphinx/docker/Dockerfile

@@ -1,6 +1,6 @@
 # Start with an image that has what we need to build the documentation
 # A better image would be something with the wheels, but not named as the base grpc-api image
-FROM deephaven/grpc-api-base
+FROM deephaven/runtime-base
 
 # install sphinx
 RUN set -eux; \

sphinx/sphinx.gradle

@@ -26,7 +26,7 @@ if (!PyEnv.pythonEnabled(project)) {
             }
         }
 //        dockerfile = project.file('Dockerfile') // not needed with the above from 'docker'
-        parentContainers = [project(':grpc-api-base').tasks.findByName('buildDocker')] // deephaven/grpc-api-base
+        parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
         imageName = 'deephaven/sphinx'
         containerOutPath = '/build'
         copyOut {
@@ -37,7 +37,7 @@ if (!PyEnv.pythonEnabled(project)) {
 //TODO core#430 remove this path entirely
 if (java8 && PyEnv.pythonEnabled(project)) {
 
-    PyEnv env = PyEnv.getEnvs(project)
+    PyEnv env = PyEnv.getEnv(project)
     /**
      * This task converts the already-expanded README file into the notebook format
      */

web/client-ide/docker/Dockerfile

@@ -2,6 +2,8 @@ FROM deephaven/iriside as iriside
 
 FROM deephaven/js-out as js-out
 
+FROM deephaven/deephaven-wheel-docs as py-docs
+
 FROM docker.io/library/nginx:1.19.7
 COPY --from=iriside /usr/src/app/package/build /usr/share/nginx/html/ide
 COPY --from=js-out /usr/src/app/raw-js-openapi/build/js-out /usr/share/nginx/html/jsapi