JPY integration test should run in docker (deephaven#591)

This fixes all non-junit tasks that are part of the jpy-integration
project, so that they run during a normal test build, including the
path/debug mains.

Partial deephaven#430

buildSrc/src/main/groovy/Docker.groovy

@@ -139,6 +139,12 @@ class Docker {
          * as-is.
          */
         List<String> entrypoint;
+
+        /**
+         * Logs are always printed from the build task when it runs, but entrypoint logs are only printed
+         * when it fails. Set this flag to always show logs, even when entrypoint is successful.
+         */
+        boolean showLogsOnSuccess;
     }
 
     /**
@@ -269,18 +275,39 @@ class Docker {
         TaskProvider<DockerLogsContainer> containerLogs = project.tasks.register("${taskName}LogsContainer", DockerLogsContainer) { logsContainer ->
             logsContainer.with {
                 containerId.set(dockerContainerName)
+                dependsOn containerFinished
                 onlyIf {
-                    cfg.entrypoint && containerFinished.get().exitCode != 0
+                    cfg.entrypoint && (containerFinished.get().exitCode != 0 || cfg.showLogsOnSuccess)
+                }
+            }
+        }
+
+        if (!cfg.copyOut) {
+            // make a wrap-up task to clean up the task work, wait until things are finished, since we have nothing to copy out
+            return project.tasks.register(taskName) { task ->
+                task.with {
+                    if (cfg.entrypoint) {
+                        dependsOn containerFinished, containerLogs
+                        doLast {
+                            // there was an entrypoint specified, if the command was not successful kill the build once
+                            // we're done copying output
+                            if (containerFinished.get().exitCode != 0) {
+                                throw new GradleException("Command '${cfg.entrypoint.join(' ')}' failed with exit code ${containerFinished.get().exitCode}, check logs for details")
+                            }
+                        }
+                    } else {
+                        dependsOn createContainer
+                    }
+                    finalizedBy removeContainer
                 }
             }
         }
-        containerFinished.configure { waitCommand -> waitCommand.finalizedBy(containerLogs) }
 
         // Copy the results from the build out of the container, so the sync task can make it available
         TaskProvider<DockerCopyFileFromContainer> copyGenerated = project.tasks.register("${taskName}CopyGeneratedOutput", DockerCopyFileFromContainer) { copy ->
             copy.with {
                 if (cfg.entrypoint) {
-                    dependsOn containerFinished
+                    dependsOn containerFinished, containerLogs
                 } else {
                     dependsOn createContainer
                 }

py/jpy-integration/build.gradle

@@ -3,6 +3,7 @@ import org.gradle.internal.jvm.Jvm
 plugins {
   id 'java'
   id 'idea'
+  id 'com.bmuschko.docker-remote-api'
 }
 
 evaluationDependsOn ':deephaven-jpy'
@@ -82,54 +83,131 @@ idea {
   }
 }
 
-// only enable this if py building is enabled
-if (PyEnv.pythonEnabled(project)) {
+Task testJavaToPython = tasks.create 'testJavaToPython', {
+  Task t ->
+    t.group = 'python'
+    t.description = 'Run the java test suite that depends on deephaven-jpy (org.jpy.PyLib, etc)'
+    check.dependsOn t
+}
 
-  PyEnv env = PyEnv.getEnv(project)
+tasks.create 'debugJavaToPython', {
+  Task t ->
+    t.group = 'python'
+    t.description = 'Run the java exec io.deephaven.jpy.integration.PyDebug which is a sanity test for all JavaExec tasks'
+}
 
-  List<Task> pythonTests = []
-  Task testJavaToPython = tasks.create 'testJavaToPython', {
-    Task t ->
-      t.group = 'python'
-      t.description = 'Run the java test suite that depends on deephaven-jpy (org.jpy.PyLib, etc)'
-      check.dependsOn t
-      pythonTests += t
-  }
+tasks.create 'pathJavaToPython', {
+  Task t ->
+    t.group = 'python'
+    t.description = 'Run the java exec io.deephaven.jpy.integration.PySysPath which prints java properties and python paths'
+}
 
-  tasks.create 'debugJavaToPython', {
-    Task t ->
-      t.group = 'python'
-      t.description = 'Run the java exec io.deephaven.jpy.integration.PyDebug which is a sanity test for all JavaExec tasks'
-  }
+Task pythonToJava = tasks.create 'unittestPythonToJava', {
+  Task t ->
+    t.group = 'python'
+    t.description = 'Run "python -m xmlrunner discover" in src/python-to-java/python'
+    check.dependsOn t
+}
 
-  tasks.create 'pathJavaToPython', {
-    Task t ->
-      t.group = 'python'
-      t.description = 'Run the java exec io.deephaven.jpy.integration.PySysPath which prints java properties and python paths'
-  }
+if (!PyEnv.pythonEnabled(project)) {
+//  Task testTask = venvTest.javaTest(project, "java-to-python-test-${pv.name}", installWheels, sourceSets.javaToPython)
+//  testJavaToPython.dependsOn testTask
+
+  Closure<TaskProvider<Task>> javaMainInDocker = { String taskName, String javaMain, SourceSet sourceSet, List<String> jvmArgs = [] ->
+    return Docker.registerDockerTask(project, taskName) {
+      copyIn {
+        from(sourceSet.runtimeClasspath) {
+          into 'classpath'
+        }
+      }
+      parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+      imageName = 'deephaven/jpy-integration-java-to-python-tests'
+      dockerfile {
+        from 'deephaven/runtime-base'
 
-  tasks.create 'debugPythonToJava', {
-    Task t ->
-      t.group = 'python'
-      t.description = 'Run the python exec python/java_debug.py which is a sanity test for all python Exec tasks in this project'
+        copyFile 'classpath', '/classpath'
+      }
+      entrypoint = ['java', '-cp', '/classpath:/classpath/*',
+                    '-Djpy.jpyLib=/usr/local/lib/python3.7/dist-packages/jpy.cpython-37m-x86_64-linux-gnu.so',
+                    '-Djpy.jdlLib=/usr/local/lib/python3.7/dist-packages/jdl.cpython-37m-x86_64-linux-gnu.so',
+                    '-Djpy.pythonLib=/usr/lib/x86_64-linux-gnu/libpython3.7m.so.1.0'
+      ] + jvmArgs + javaMain
+
+      showLogsOnSuccess = true
+    }
   }
 
-  Task pythonToJava = tasks.create 'unittestPythonToJava', {
-    Task t ->
-      t.group = 'python'
-      t.description = 'Run "python -m xmlrunner discover" in src/python-to-java/python'
-      check.dependsOn t
-      pythonTests += t
+  debugJavaToPython.dependsOn javaMainInDocker('java-to-python-debug', 'io.deephaven.jpy.integration.PyDebug', sourceSets.javaToPython, ["-Djpy.debug=true"])
+
+  pathJavaToPython.dependsOn javaMainInDocker('java-to-python-path', 'io.deephaven.jpy.integration.PySysPath', sourceSets.javaToPython)
+
+
+  Sync jpyConfigTask = tasks.create 'createPythonToJavaConfig', Sync, {
+    // if the classpath changes, we need to rebuild this
+    inputs.files sourceSets.pythonToJava.runtimeClasspath
+    from('src/pythonToJava/jpyconfig.py.template')
+    into "${buildDir}/jpyConfig"
+    rename { file -> 'jpyconfig.py' }
+    doFirst {
+      expand(
+              javaHome:'/opt/java/openjdk',
+              jvmMaxMem:'32m',
+              jvmClasspath:'/classpath:' + sourceSets.pythonToJava.runtimeClasspath.asList().collect { '/classpath/' + it.name }.join(':')
+      )
+    }
   }
 
-  tasks.create 'testPython', {
-    Task t ->
-      t.group = 'python'
-      t.description = 'Run all python-related tests in the jpy-integration module'
-      check.dependsOn t
-      t.dependsOn(pythonTests)
+  Closure<TaskProvider<Task>> pyExec = { String taskName, List<String> command, boolean showLogs = false ->
+    File reportDir = project.file("${buildDir}/test-results/${taskName}")
+    return Docker.registerDockerTask(project, taskName) {
+      copyIn {
+        from ('src/pythonToJava/python') {
+          into 'python'
+        }
+        from (sourceSets.pythonToJava.runtimeClasspath) {
+          into 'classpath'
+        }
+        from(jpyConfigTask.outputs.files) {
+          into 'python'
+        }
+      }
+      parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+      imageName = 'deephaven/jpy-integration-python-to-java-tests'
+      dockerfile {
+        // set up the container, env vars - things that aren't likely to change
+        from 'deephaven/runtime-base'
+        runCommand '''set -eux; \\
+                        pip3 install unittest-xml-reporting==3.0.4;\\
+                        mkdir /out;'''
+        environmentVariable 'JDK_HOME', '/opt/java/openjdk'
+        environmentVariable 'JAVA_VERSION', '1.8'
+        environmentVariable 'JPY_PY_CONFIG', '/python/jpyconfig.py'
+
+        workingDir '/python'
+
+        // copy in the contents that we do expect to change as the project updates
+        copyFile 'python', '/python'
+        copyFile 'classpath', '/classpath'
+      }
+      entrypoint = command
+
+      copyOut {
+        into reportDir
+      }
+      showLogsOnSuccess = showLogs
+    }
   }
 
+  pythonToJava.dependsOn pyExec('unittest-python-to-java', ['python3', '-m', 'xmlrunner', 'discover', '-v', '-o', '/out/'])
+
+  // These tests need to run in their own process, since jpy can only start up a jvm once per process.
+  pythonToJava.dependsOn pyExec('unittest-single-jpy-python-to-java', ['python3', '-m', 'xmlrunner', '-v', '-o', '/out/', 'test.main_single_jpy_create_destroy_jvm'])
+  pythonToJava.dependsOn pyExec('unittest-multiple-jpy-python-to-java', ['python3', '-m', 'xmlrunner', '-v', '-o', '/out/', 'test.main_multiple_jpy_create_destroy_jvm'])
+
+  pythonToJava.dependsOn pyExec('debugPythonToJava', ['python3', 'java_debug.py'], true)
+} else {
+  // only enable this if py building is enabled
+
   File genConfig = file("$buildDir/pyGen/jpyconfig.py")
   Sync jpyConfigTask = tasks.create 'createPythonToJavaConfig', Sync, {
     Sync s ->
@@ -153,6 +231,7 @@ if (PyEnv.pythonEnabled(project)) {
       }
   }
 
+  PyEnv env = PyEnv.getEnv(project)
   for (pv in [PythonVersion.PY_37]) {
     Venv venvTest = env.getVenv(pv, VenvType.TEST_JPY)
 

py/jpy-integration/src/javaToPython/java/io/deephaven/jpy/integration/PyDebug.java

@@ -17,8 +17,10 @@ private static void debug(String log) {
 
   public static void main(String[] args) {
 
+    JpyConfigExt jpyConfigExt = new JpyConfigExt(SysProps.INSTANCE.asJpyConfig());
+    jpyConfigExt.initPython();
     debug("Calling JpyConfig.startPython()...");
-    new JpyConfigExt(SysProps.INSTANCE.asJpyConfig()).startPython();
+    jpyConfigExt.startPython();
     debug("Called JpyConfig.startPython()");
 
     debug("Called PyLib.isPythonRunning() = " + PyLib.isPythonRunning());