diff --git a/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java b/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java index f37f58e3907..59c03d1551d 100644 --- a/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java +++ b/Integrations/src/main/java/io/deephaven/integrations/python/PythonDeephavenSession.java @@ -52,9 +52,6 @@ /** * A ScriptSession that uses a JPy cpython interpreter internally. - *

- * This is used for applications or the console; Python code running remotely uses WorkerPythonEnvironment for it's - * supporting structures. */ public class PythonDeephavenSession extends AbstractScriptSession { private static final Logger log = LoggerFactory.getLogger(PythonDeephavenSession.class); diff --git a/python-engine-test/build.gradle b/python-engine-test/build.gradle deleted file mode 100644 index 8c24a953e6a..00000000000 --- a/python-engine-test/build.gradle +++ /dev/null @@ -1,82 +0,0 @@ -plugins { - id 'com.bmuschko.docker-remote-api' - id 'io.deephaven.project.register' -} - -evaluationDependsOn ':docker-runtime-base' // runtimeBase - -configurations { - testImplementation.extendsFrom fishIoTest -} - -test { - exclude "**/*" // make sure IntelliJ doesn't assume this test task is the one to use to run classes. - onlyIf { false } // don't run tests, we'll set up ones for each python env. -} - -dependencies { - testImplementation project(':engine-context') - testImplementation project(':engine-table') - testImplementation project(':Integrations') - testImplementation TestTools.projectDependency(project, 'engine-table') - - // This allows us to access jpy-integration javaToPython code around setting up python - // for testing purposes - testImplementation project.dependencies.project([ - path: ':jpy-integration', - configuration: 'javaToPythonJar' - ]) - - // needed for access to dh-tests.prop - testRuntimeOnly project(path: ':test-configs') - testRuntimeOnly project(path: ':configs') - testRuntimeOnly project(':Integrations') - testRuntimeOnly project(':engine-table') - - // todo: IDO-322 (io.deephaven.numerics.suanshu.SuanShuIntegration) - testRuntimeOnly project(':Numerics') - testRuntimeOnly project(':extensions-suanshu') -} - -def gradleWrapper = tasks.register("dockerGradleInit", Wrapper.class) { wrapper -> - wrapper.scriptFile "${buildDir}/template-project/gradlew" - wrapper.jarFile "${buildDir}/template-project/gradle/wrapper/gradle-wrapper.jar" - wrapper.gradleVersion '8.4' -} - -tasks.getByName('check').dependsOn Docker.registerDockerTask(project, 'test-in-docker') { - copyIn { - dependsOn gradleWrapper - from ("${buildDir}/template-project") { - into 'project' - } - from(project.file('src/test/build.gradle.template')) { - into 'project' - rename { file -> 'build.gradle' } - } - from (sourceSets.test.runtimeClasspath) { - into 'classpath' - } - from (sourceSets.test.output.getClassesDirs()) { - into 'classes' - } - } - parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base - dockerfile { - // base image with default java, python, wheels - from 'deephaven/runtime-base:local-build' - - // set up the project - copyFile 'project', '/project' - workingDir '/project' - // run once with no actual classes, so that gradle is preinstalled and cached - runCommand '/project/gradlew --version' - copyFile 'classpath', '/classpath' - copyFile 'classes', '/classes' - } - entrypoint = ['/project/gradlew', 'test', '--info'] - containerOutPath = '/project/build/test-results/test/' - copyOut { - into "${buildDir}/test-results/test-in-docker" - } -} diff --git a/python-engine-test/gradle.properties b/python-engine-test/gradle.properties deleted file mode 100644 index eeac3e65888..00000000000 --- a/python-engine-test/gradle.properties +++ /dev/null @@ -1 +0,0 @@ -io.deephaven.project.ProjectType=JAVA_LOCAL diff --git a/python-engine-test/src/test/build.gradle.template b/python-engine-test/src/test/build.gradle.template deleted file mode 100644 index f8d797df425..00000000000 --- a/python-engine-test/src/test/build.gradle.template +++ /dev/null @@ -1,19 +0,0 @@ -plugins { - id 'java-library' -} - -// Classpath is already built, just need to pass to the test task -test.classpath = fileTree('/classpath/').plus(files('/classpath')) -test.testClassesDirs = files('/classes') - -test.systemProperties([ - 'jpy.jpyLib':'/opt/deephaven-venv/lib/python3.7/site-packages/jpy.cpython-37m-x86_64-linux-gnu.so', - 'jpy.jdlLib':'/opt/deephaven-venv/lib/python3.7/site-packages/jdl.cpython-37m-x86_64-linux-gnu.so', - 'jpy.pythonLib':'/usr/lib/x86_64-linux-gnu/libpython3.7m.so.1.0', - // Cleaning up on a dedicated thread has some issues when there is frequent starting - // and stopping of the python virtual environment. We'll rely on cleaning up inline - // when necessary. - // TODO issue #651 to see if we can remove this - 'PyObject.cleanup_on_thread':'false', -// 'jpy.debug':'true' -]) diff --git a/python-engine-test/src/test/java/io/deephaven/engine/table/impl/select/PythonMatchFilterTest.java b/python-engine-test/src/test/java/io/deephaven/engine/table/impl/select/PythonMatchFilterTest.java deleted file mode 100644 index 0ae82bdfc38..00000000000 --- a/python-engine-test/src/test/java/io/deephaven/engine/table/impl/select/PythonMatchFilterTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.table.impl.select; - -import static org.junit.Assert.assertEquals; - -import io.deephaven.configuration.Configuration; -import io.deephaven.engine.context.QueryScope; -import io.deephaven.io.log.LogLevel; -import io.deephaven.io.logger.StreamLoggerImpl; -import io.deephaven.util.process.ProcessEnvironment; -import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.util.WorkerPythonEnvironment; -import io.deephaven.jpy.PythonTest; - -import java.util.List; - -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Test MatchFilters that reference Python lists. - */ -@Ignore // TODO (deephaven-core#734) -public class PythonMatchFilterTest extends PythonTest { - - @Before - public void setUp() { - if (ProcessEnvironment.tryGet() == null) { - ProcessEnvironment.basicInteractiveProcessInitialization(Configuration.getInstance(), - PythonMatchFilterTest.class.getCanonicalName(), new StreamLoggerImpl(System.out, LogLevel.INFO)); - } - } - - @Test - public void testIntMatch() { - WorkerPythonEnvironment.DEFAULT.eval("iii = [1, 2, 3]"); - Object iii = WorkerPythonEnvironment.DEFAULT.fetch("iii"); - - QueryScope.addParam("iii", iii); - WhereFilter filter = WhereFilterFactory.getExpression("ival in iii"); - assertEquals(MatchFilter.class, filter.getClass()); - - TableDefinition tableDef = TableDefinition.from(List.of("ival"), List.of(int.class)); - filter.init(tableDef); - Object[] values = ((MatchFilter) filter).getValues(); - // System.out.println(Arrays.toString(values)); - assertEquals(1, values[0]); - assertEquals(2, values[1]); - assertEquals(3, values[2]); - } - - @Test - public void testStrMatch() { - WorkerPythonEnvironment.DEFAULT.eval("ss = [\"aa\", \"bb\", \"cc\"]"); - Object ss = WorkerPythonEnvironment.DEFAULT.fetch("ss"); - - QueryScope.addParam("ss", ss); - WhereFilter filter = WhereFilterFactory.getExpression("sval in ss"); - assertEquals(MatchFilter.class, filter.getClass()); - - TableDefinition tableDef = TableDefinition.from(List.of("sval"), List.of(String.class)); - filter.init(tableDef); - Object[] values = ((MatchFilter) filter).getValues(); - // System.out.println(Arrays.toString(values)); - assertEquals("aa", values[0]); - assertEquals("bb", values[1]); - assertEquals("cc", values[2]); - } -} diff --git a/python-engine-test/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilter.java b/python-engine-test/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilter.java deleted file mode 100644 index 9c3c1f38ab9..00000000000 --- a/python-engine-test/src/test/java/io/deephaven/engine/table/impl/select/TestConditionFilter.java +++ /dev/null @@ -1,446 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.table.impl.select; - -import io.deephaven.base.verify.Require; -import io.deephaven.configuration.Configuration; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.RowSetBuilderSequential; -import io.deephaven.engine.rowset.RowSetFactory; -import io.deephaven.io.log.LogLevel; -import io.deephaven.io.logger.StreamLoggerImpl; -import io.deephaven.util.process.ProcessEnvironment; -import io.deephaven.engine.context.QueryCompiler; -import io.deephaven.engine.table.Table; -import io.deephaven.engine.context.QueryScopeParam; -import io.deephaven.engine.context.QueryScope; -import io.deephaven.integrations.python.PythonDeephavenSession; -import io.deephaven.engine.util.PythonScopeJpyImpl; -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.jpy.PythonTest; -import io.deephaven.util.thread.ThreadInitializationFactory; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.jpy.PyInputMode; -import org.jpy.PyModule; -import org.jpy.PyObject; -import org.junit.After; -import org.junit.Assert; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Predicate; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import static io.deephaven.engine.table.impl.select.FormulaTestUtil.*; -import static org.jpy.PyLib.getMainGlobals; -import static org.junit.Assert.fail; - -@Ignore // TODO (deephaven-core#734) -public class TestConditionFilter extends PythonTest { - static { - if (ProcessEnvironment.tryGet() == null) { - ProcessEnvironment.basicInteractiveProcessInitialization(Configuration.getInstance(), - TestConditionFilter.class.getCanonicalName(), new StreamLoggerImpl(System.out, LogLevel.INFO)); - } - } - - private static final boolean ENABLE_QUERY_COMPILER_LOGGING = Configuration.getInstance() - .getBooleanForClassWithDefault(TestConditionFilter.class, "QueryCompiler.logEnabled", false); - - private final Table testDataTable; - private boolean queryCompilerLogEnabledInitial = false; - - public TestConditionFilter() { - testDataTable = getTestDataTable(); - - setUpQueryLibrary(); - setUpQueryScope(); - } - - @Before - public void setUp() throws Exception { - if (ProcessEnvironment.tryGet() == null) { - ProcessEnvironment.basicInteractiveProcessInitialization(Configuration.getInstance(), - PythonMatchFilterTest.class.getCanonicalName(), new StreamLoggerImpl(System.out, LogLevel.INFO)); - } - queryCompilerLogEnabledInitial = QueryCompiler.setLogEnabled(ENABLE_QUERY_COMPILER_LOGGING); - } - - @After - public void tearDown() throws Exception { - QueryCompiler.setLogEnabled(queryCompilerLogEnabledInitial); - } - - @Test - public void testTrueFalse() { - String expression; - Predicate> test; - - expression = "true"; - test = (colValues) -> true; - check(expression, test, true); - - expression = "false"; - test = (colValues) -> false; - check(expression, test, true); - } - - @Test - public void testObjectConstruction() { - String expression; - Predicate> test; - - expression = "new Boolean(true)"; - test = (colValues) -> true; - check(expression, test); - - expression = "new Boolean(false)"; - test = (colValues) -> false; - check(expression, test); - } - - @Test - public void testRuntimeException() { - checkExpectingEvaluationException("((Boolean) null)", "NullPointerException"); - checkExpectingEvaluationException("Integer.parseInt(\"this is not an integer\") != null", - "NumberFormatException"); - } - - @Test - public void testBadExpressionType() { - checkExpectingCompilationException("0", "boolean required"); - checkExpectingCompilationException("IntCol", "boolean required"); - } - - @Test - public void testMiscCompilationExceptions() { - checkExpectingCompilationException("nonExistentVariableOrClass", - "Cannot find variable or class nonExistentVariableOrClass"); - checkExpectingCompilationException("Integer.noSuchMethod()", "Cannot find method noSuchMethod()"); - } - - @Test - public void testNullFilters() { - String expression = "!isNull(IntCol)"; - Predicate> test = (colValues) -> colValues.get("IntCol") != null; - check(expression, test); - - expression = "isNull(IntCol)"; - test = (colValues) -> colValues.get("IntCol") == null; - check(expression, test); - } - - @Test - public void testComparison() { - String expression; - Predicate> test; - - { // LESS THAN - expression = "myShortObj < ShortCol"; - test = (colValues) -> QueryLanguageFunctionUtils.less( - QueryLanguageFunctionUtils.shortCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.shortCast(colValues.get("ShortCol"))); - check(expression, test, true, false); - - expression = "myIntObj < IntCol"; - test = (colValues) -> QueryLanguageFunctionUtils.less( - QUERYSCOPE_OBJ_BASE_VALUE, - QueryLanguageFunctionUtils.intCast(colValues.get("IntCol"))); - check(expression, test, true); - - expression = "myLongObj < LongCol"; - test = (colValues) -> QueryLanguageFunctionUtils.less( - QueryLanguageFunctionUtils.longCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.longCast(colValues.get("LongCol"))); - check(expression, test, true); - - expression = "myFloatObj < FloatCol"; - test = (colValues) -> QueryLanguageFunctionUtils.less( - QueryLanguageFunctionUtils.floatCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.floatCast(colValues.get("FloatCol"))); - check(expression, test, true); - - expression = "myDoubleObj < DoubleCol"; - test = (colValues) -> QueryLanguageFunctionUtils.less( - QueryLanguageFunctionUtils.doubleCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.doubleCast(colValues.get("DoubleCol"))); - check(expression, test, true); - } - - { // GREATER THAN - expression = "myShortObj > ShortCol"; - test = (colValues) -> QueryLanguageFunctionUtils.greater( - QueryLanguageFunctionUtils.shortCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.shortCast(colValues.get("ShortCol"))); - check(expression, test, true); - - expression = "myIntObj > IntCol"; - test = (colValues) -> QueryLanguageFunctionUtils.greater( - QUERYSCOPE_OBJ_BASE_VALUE, - QueryLanguageFunctionUtils.intCast(colValues.get("IntCol"))); - check(expression, test, true); - - expression = "myLongObj > LongCol"; - test = (colValues) -> QueryLanguageFunctionUtils.greater( - QueryLanguageFunctionUtils.longCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.longCast(colValues.get("LongCol"))); - check(expression, test, true); - - expression = "myFloatObj > FloatCol"; - test = (colValues) -> QueryLanguageFunctionUtils.greater( - QueryLanguageFunctionUtils.floatCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.floatCast(colValues.get("FloatCol"))); - check(expression, test, true); - - expression = "myDoubleObj > DoubleCol"; - test = (colValues) -> QueryLanguageFunctionUtils.greater( - QueryLanguageFunctionUtils.doubleCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.doubleCast(colValues.get("DoubleCol"))); - check(expression, test, true); - } - - { // EQUAL - expression = "myShortObj == ShortCol"; - test = (colValues) -> QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.shortCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.shortCast(colValues.get("ShortCol"))); - check(expression, test, true); - - expression = "myIntObj == IntCol"; - test = (colValues) -> QueryLanguageFunctionUtils.eq( - QUERYSCOPE_OBJ_BASE_VALUE, - QueryLanguageFunctionUtils.intCast(colValues.get("IntCol"))); - check(expression, test, true); - - expression = "myLongObj == LongCol"; - test = (colValues) -> QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.longCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.longCast(colValues.get("LongCol"))); - check(expression, test, true); - - expression = "myFloatObj == FloatCol"; - test = (colValues) -> QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.floatCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.floatCast(colValues.get("FloatCol"))); - check(expression, test, true); - - expression = "myDoubleObj == DoubleCol"; - test = (colValues) -> QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.doubleCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.doubleCast(colValues.get("DoubleCol"))); - check(expression, test, true); - } - - { // NOT EQUAL - expression = "myShortObj != ShortCol"; - test = (colValues) -> !QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.shortCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.shortCast(colValues.get("ShortCol"))); - check(expression, test, true); - - expression = "myIntObj != IntCol"; - test = (colValues) -> !QueryLanguageFunctionUtils.eq( - QUERYSCOPE_OBJ_BASE_VALUE, - QueryLanguageFunctionUtils.intCast(colValues.get("IntCol"))); - check(expression, test, true); - - expression = "myLongObj != LongCol"; - test = (colValues) -> !QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.longCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.longCast(colValues.get("LongCol"))); - check(expression, test, true); - - expression = "myFloatObj != FloatCol"; - test = (colValues) -> !QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.floatCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.floatCast(colValues.get("FloatCol"))); - check(expression, test, true); - - expression = "myDoubleObj != DoubleCol"; - test = (colValues) -> !QueryLanguageFunctionUtils.eq( - QueryLanguageFunctionUtils.doubleCast(QUERYSCOPE_OBJ_BASE_VALUE), - QueryLanguageFunctionUtils.doubleCast(colValues.get("DoubleCol"))); - check(expression, test, true); - } - } - - @Test - public void testLoadNumpyTwice() { - Assert.assertNotNull(PyModule.importModule("deephaven/numba")); - Assert.assertNotNull(PyModule.importModule("numpy")); - Assert.assertNotNull(PyModule.importModule("deephaven.lang.vectorize_simple")); - Assert.assertNotNull(PyModule.importModule("deephaven/numba")); - Assert.assertNotNull(PyModule.importModule("numpy")); - Assert.assertNotNull(PyModule.importModule("deephaven.lang.vectorize_simple")); - } - - @Test - public void testPython() { - PyObject.executeCode("from numba.npyufunc import vectorize\n" + - "@vectorize\n" + - "def testf(a, b, c):\n" + - " return a + b < c\n" + - "\n", PyInputMode.SCRIPT); - - check("testf(IntCol,IntCol*2,IntCol+2)", m -> { - Integer ic = (Integer) m.get("IntCol"); - if (ic == null) { - return true; - } - return ic + ic * 2 < ic + 2; - }, true, false); - } - - @Test - public void testIIIK() { - - check("i > 1", m -> { - Integer i = (Integer) m.get("actualI"); - return i > 1; - }, true, false); - check("i <= 1", m -> { - Integer i = (Integer) m.get("actualI"); - return i <= 1; - }, true, false); - check("k > 1", m -> { - Long k = (Long) m.get("actualK"); - return k > 1; - }, true, false); - check("k <= 1", m -> { - Long k = (Long) m.get("actualK"); - return k <= 1; - }, true, false); - check("ii > 1", m -> { - Long ii = (Long) m.get("actualII"); - return ii > 1; - }, true, false); - check("ii <= 1", m -> { - Long ii = (Long) m.get("actualII"); - return ii <= 1; - }, true, false); - } - - - /** - * Ensure that a {@link ConditionFilter} with the given {@code expression} {@link #testDataTable} filtered by a - * ConditionF - * - * @param expression the conditional expression to check - * @param testPredicate the predicate over a map of column values to compare with the expression - */ - private void check(String expression, Predicate> testPredicate) { - check(expression, testPredicate, false, true); - } - - private void check(String expression, Predicate> testPredicate, boolean testPython) { - check(expression, testPredicate, testPython, true); - } - - private void check(String expression, Predicate> testPredicate, boolean testPython, - boolean testNative) { - final RowSetBuilderSequential keepBuilder = RowSetFactory.builderSequential(); - final RowSetBuilderSequential dropBuilder = RowSetFactory.builderSequential(); - - final Map sourcesMap = - testDataTable.updateView("actualI = i", "actualII = ii", "actualK = k").getColumnSourceMap(); - - for (final RowSet.Iterator it = testDataTable.getRowSet().iterator(); it.hasNext();) { - final long idx = it.nextLong(); - final Map rowMap = new HashMap<>(sourcesMap.size()); - for (Map.Entry entry : sourcesMap.entrySet()) { - rowMap.put( - entry.getKey(), - entry.getValue().get(idx)); - } - if (testPredicate.test(rowMap)) { - keepBuilder.appendKey(idx); - } else { - dropBuilder.appendKey(idx); - } - } - - final RowSet keepRowSet = keepBuilder.build(); - final RowSet dropRowSet = dropBuilder.build(); - - if (testNative) { - validate(expression, keepRowSet, dropRowSet, FormulaParserConfiguration.Deephaven); - } - if (testPython) { - ExecutionContext currentContext = ExecutionContext.getContext(); - QueryScope currentScope = currentContext.getQueryScope(); - try { - if (pythonScope == null) { - final ExecutionContext context = new PythonDeephavenSession( - ExecutionContext.getDefaultContext().getUpdateGraph(), - ThreadInitializationFactory.NO_OP, - new PythonScopeJpyImpl(getMainGlobals().asDict())).getExecutionContext(); - pythonScope = context.getQueryScope(); - context.open(); - } - for (QueryScopeParam param : currentScope.getParams(currentScope.getParamNames())) { - pythonScope.putParam(param.getName(), param.getValue()); - } - expression = expression.replaceAll("true", "True").replaceAll("false", "False"); - validate(expression, keepRowSet, dropRowSet, FormulaParserConfiguration.Numba); - } finally { - currentContext.open(); - } - } - - } - - private void validate(String expression, RowSet keepRowSet, RowSet dropRowSet, FormulaParserConfiguration parser) { - final RowSet filteredRowSet = initCheck(expression, parser); - - Require.eq(keepRowSet.size(), "keepRowSet.size()", filteredRowSet.size(), "filteredRowSet.size()"); - Require.eq(keepRowSet.intersect(filteredRowSet).size(), "keepRowSet.intersect(filteredRowSet).size()", - filteredRowSet.size(), "filteredRowSet.size()"); - Require.eqZero(dropRowSet.intersect(filteredRowSet).size(), "dropRowSet.intersect(filteredRowSet).size()"); - } - - - private void checkExpectingEvaluationException(String expression, String expectedCauseMessage) { - try { - initCheck(expression, FormulaParserConfiguration.Deephaven); - fail("Should have thrown an exception"); - } catch (FormulaEvaluationException ex) { - if (!ex.getMessage().contains(expectedCauseMessage) - && !ex.getCause().getMessage().contains(expectedCauseMessage)) // check the cause, since all - // exceptions during filter - // evaluation are caught - { - fail("Useless exception message!\nOriginal exception:\n" + ExceptionUtils.getStackTrace(ex)); - } - } - } - - private void checkExpectingCompilationException(String expression, String expectedCauseMessage) { - try { - initCheck(expression, FormulaParserConfiguration.Deephaven); - fail("Should have thrown an exception"); - } catch (FormulaCompilationException ex) { - if (!ex.getMessage().contains(expectedCauseMessage) - && !ex.getCause().getMessage().contains(expectedCauseMessage)) // check the cause, since all - // exceptions during filter init are - // caught - { - fail("Useless exception message!\nOriginal exception:\n" + ExceptionUtils.getStackTrace(ex)); - } - } - - } - - - private RowSet initCheck(String expression, FormulaParserConfiguration parser) { - final WhereFilter conditionFilter = ConditionFilter.createConditionFilter(expression, parser); - conditionFilter.init(testDataTable.getDefinition()); - return conditionFilter.filter(testDataTable.getRowSet().copy(), testDataTable.getRowSet(), testDataTable, - false); - } - -} diff --git a/python-engine-test/src/test/java/io/deephaven/engine/util/PickledResult.java b/python-engine-test/src/test/java/io/deephaven/engine/util/PickledResult.java deleted file mode 100644 index 92e03810228..00000000000 --- a/python-engine-test/src/test/java/io/deephaven/engine/util/PickledResult.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.util; - -import java.io.Serializable; - -/** - * Pickled result for a Python fetch. - */ -public class PickledResult implements Serializable { - private String pickled; - private String pythonVersion; // version of python used to perform pickle - - PickledResult(String pickled, String pythonVersion) { - this.pickled = pickled; - this.pythonVersion = pythonVersion; - } - - public String getPickled() { - return pickled; - } - - public String getPythonVersion() { - return pythonVersion; - } -} diff --git a/python-engine-test/src/test/java/io/deephaven/engine/util/TestWorkerPythonEnvironment.java b/python-engine-test/src/test/java/io/deephaven/engine/util/TestWorkerPythonEnvironment.java deleted file mode 100644 index 7aa7ad0c274..00000000000 --- a/python-engine-test/src/test/java/io/deephaven/engine/util/TestWorkerPythonEnvironment.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.util; - -import io.deephaven.configuration.Configuration; -import io.deephaven.engine.testutil.testcase.RefreshingTableTestCase; -import io.deephaven.io.log.LogLevel; -import io.deephaven.io.logger.StreamLoggerImpl; -import io.deephaven.util.process.ProcessEnvironment; -import io.deephaven.engine.table.Table; -import io.deephaven.engine.util.jpy.JpyInit; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -import org.junit.Ignore; - -/** - * Test various Jpy related overloading methods. - */ -@Ignore // TODO (deephaven-core#734) -public class TestWorkerPythonEnvironment extends RefreshingTableTestCase { - - @Override - public void setUp() throws Exception { - super.setUp(); - if (ProcessEnvironment.tryGet() == null) { - ProcessEnvironment.basicInteractiveProcessInitialization(Configuration.getInstance(), - TestWorkerPythonEnvironment.class.getCanonicalName(), - new StreamLoggerImpl(System.out, LogLevel.INFO)); - } - } - - public void testNumpyImport() { - WorkerPythonEnvironment.DEFAULT.eval("import numpy"); - } - - public void testTimeTable() throws IOException { - WorkerPythonEnvironment.DEFAULT.eval("tt = timeTable(\"PT00:00:01\")"); - Object result = WorkerPythonEnvironment.DEFAULT.getValue("tt"); - assertTrue(result instanceof Table); - Table tt = (Table) result; - TableTools.show(tt); - } - - public void testEmptyTable() throws IOException { - WorkerPythonEnvironment.DEFAULT - .eval("TableTools = jpy.get_type(\"io.deephaven.engine.util.TableTools\")"); - WorkerPythonEnvironment.DEFAULT.eval("et = TableTools.emptyTable(2).update(\"A=k\")"); - Object result = WorkerPythonEnvironment.DEFAULT.getValue("et"); - assertTrue(result instanceof Table); - Table et = (Table) result; - TableTools.show(et); - } - - public void testUpdateList() throws IOException { - WorkerPythonEnvironment.DEFAULT - .eval("TableTools = jpy.get_type(\"io.deephaven.engine.util.TableTools\")"); - WorkerPythonEnvironment.DEFAULT.eval("et = TableTools.emptyTable(2).update([\"A=k\", \"B=i*2\"])"); - Object result = WorkerPythonEnvironment.DEFAULT.getValue("et"); - assertTrue(result instanceof Table); - Table et = (Table) result; - TableTools.show(et); - } - - public void testUpdateVarArgs() throws IOException { - WorkerPythonEnvironment.DEFAULT - .eval("TableTools = jpy.get_type(\"io.deephaven.engine.util.TableTools\")"); - WorkerPythonEnvironment.DEFAULT.eval("et = TableTools.emptyTable(2).update(\"A=k\", \"B=i*2\")"); - Object result = WorkerPythonEnvironment.DEFAULT.getValue("et"); - assertTrue(result instanceof Table); - Table et = (Table) result; - TableTools.show(et); - } - - - public void testScript() throws IOException, InterruptedException, TimeoutException { - JpyInit.init(); - final PythonEvaluator evaluator = PythonEvaluatorJpy.withGlobalCopy(); - final String filename = "/tmp/_not_existent_file.py"; - - final File file = new File(filename); - assertFalse(file.exists()); - - try { - evaluator.runScript(filename); - // we should never get here - assertFalse(true); - } catch (FileNotFoundException fnfe) { - assertEquals(fnfe.getMessage(), filename); - } - - System.out.println("Run script done."); - } -} diff --git a/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java b/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java deleted file mode 100644 index cda1eb05d59..00000000000 --- a/python-engine-test/src/test/java/io/deephaven/engine/util/WorkerPythonEnvironment.java +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending - */ -package io.deephaven.engine.util; - -import io.deephaven.UncheckedDeephavenException; -import io.deephaven.base.FileUtils; -import io.deephaven.base.verify.Assert; -import io.deephaven.configuration.Configuration; -import io.deephaven.integrations.python.PythonDeephavenSession; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; -import io.deephaven.engine.context.QueryScope; -import org.jpy.PyObject; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.concurrent.TimeoutException; - -/** - * This class is the support infrastructure for running Python remote queries. - * - * It is a singleton that contains an instance of a PythonHolder. All of the specially handled engine operations from a - * remote Python session should execute queries which interact wtih this class. The script sessions that run for - * PersistentQueries or consoles are handled separately by the {@link PythonDeephavenSession}. - */ -public enum WorkerPythonEnvironment { - DEFAULT; - - private final PythonEvaluatorJpy evaluator; - private final PythonScope scope; - private final String pythonVersion; - private final String[] pythonVersionParts; - - private final Logger log; - - /** - * Create the environment using our global log and a JpyHolder. - * - * Runs the init script specified by WorkerPythonEnvironment.initScript. - */ - WorkerPythonEnvironment() { - log = LoggerFactory.getLogger(WorkerPythonEnvironment.class); - try { - evaluator = PythonEvaluatorJpy.withGlobalCopy(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new UncheckedDeephavenException(e); - } catch (TimeoutException e) { - throw new UncheckedDeephavenException(e); - } - scope = evaluator.getScope(); - pythonVersion = evaluator.getPythonVersion(); - pythonVersionParts = pythonVersion.split("\\.", 0); // NB: you have to escape "." ... - if (pythonVersionParts.length > 2) { - log.info().append("Worker python version ").append(pythonVersion).endl(); - } else { - log.warn().append("Worker python version set as ").append(pythonVersion) - .append(" which has unexpected format (not `....` which may " + - "lead to unexpected errors") - .endl(); - } - - final String defaultScriptPath = Configuration.getInstance() - .getStringWithDefault("WorkerPythonEnvironment.defaultScriptPath", "."); - - final ScriptFinder scriptFinder = new ScriptFinder(defaultScriptPath); - final String initScript = - Configuration.getInstance().getStringWithDefault("WorkerPythonEnvironment.initScript", ""); - - final ScriptFinder.FileOrStream file; - try { - file = scriptFinder.findScriptEx(initScript); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - if (file.getFile().isPresent()) { - try { - evaluator.runScript(file.getFile().get().getAbsolutePath()); - } catch (FileNotFoundException fnfe) { - throw new UncheckedIOException(fnfe); - } - } else { - Assert.assertion(file.getStream().isPresent(), "file.getStream().isPresent()"); - - final String scriptText; - try { - scriptText = FileUtils.readTextFile(file.getStream().get()); - } catch (IOException ioe) { - throw new UncheckedIOException(ioe); - } - evaluator.evalScript(scriptText); - } - } - - /** - * Retrieves a value from our Python holder's globals. - * - * When the object is a convertible PyObject; we return the PyObject. Otherwise, we'll return a - * PythonRemoteQuery.PickledResult, which is suitable for unpickling by the remote side. - * - * The caller should never serialize an unconverted PyObject; it contains a raw pointer and will result in a Hotspot - * or memory corruption on the remote side. - * - * @param name the variable to retrieve - * @return the variable as a Java object; or pickled - */ - public Object fetch(String name) { - Object x; - boolean tryPickle = false; - - x = getValue(name); - if (x instanceof PyObject) { - if (!((PyObject) x).isConvertible()) { - tryPickle = true; - } - } - - if (tryPickle) { - log.info().append("Variable ").append(name).append(" requires pickling.").endl(); - evaluator.evalStatement("__resultDill__ = dill.dumps(__result__)"); - if (pythonVersionParts[0].equals("2")) { - // In python2, we have that str is 8 bits, and base64.b64encode produces a str - evaluator.evalStatement("__resultPickled__ = base64.b64encode(__resultDill__)"); - } else { - // In python3, we have that str is 32 bit unicode, and base64.b64encode produces a bytes (array - // basically) - // our next step is to cast this output as a java string, which does not work for a bytes. - // We must make it a str, via calling .decode() on it. - evaluator.evalStatement("__resultPickled__ = base64.b64encode(__resultDill__).decode()"); - } - // this is the only place a base64 encoded item is cast to a string, so only fix needed - String pickled = (String) getValue("__resultPickled__"); - x = new PickledResult(pickled, pythonVersion); - } else { - log.info().append("Variable ").append(name).append(" is of type ").append(x.getClass().getCanonicalName()) - .endl(); - } - - return x; - } - - /** - * Evaluates the given string as a statement. - * - * @param evalString the statement to evaluate - */ - public void eval(String evalString) { - evaluator.evalStatement(evalString); - } - - /** - * Retrieves a value from the Python holder, with default conversion, but no (un)pickling. - * - * Used for unit testing. - * - * @param variable the variable name to retrieve - * @return the value from the Python holder's environment. - */ - Object getValue(String variable) { - return scope - .getValue(variable) - .orElseThrow(() -> new QueryScope.MissingVariableException("No variable: " + variable)); - } -} - diff --git a/settings.gradle b/settings.gradle index 587ce55a9d7..fdd30f42cd8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,7 +16,6 @@ String[] mods = [ 'ModelFarm', 'ClientSupport', 'BenchmarkSupport', - 'python-engine-test', 'server', 'table-api', 'qst',