From 144b677b887bf0f32d8b275e01dea52a810d522b Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Thu, 20 May 2021 10:42:15 -0600 Subject: [PATCH 01/10] Fix a link formatting bug in CONTRIBUTING.md (#629) * Fix a link formatting bug in CONTRIBUTING.md * Update CONTRIBUTING.md * Update CONTRIBUTING.md Co-authored-by: Mike Bender Co-authored-by: Mike Bender --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9e6bb3ffdd..37451190488 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ To get started quickly: 3) `git clone git@github.com:/deephaven-core.git` 4) Commit changes to your own branches in your forked repository. -Forked repositories do not have access to the same tokens/secrets as the deephaven/deephaven-core depository, so GitHub actions will fail. To disable GitHub actions in your forked repository, go to "Actions" -> "Disable Actions" in your forked repository settings (https://github.com//deephaven-core/settings/actions). +Forked repositories do not have access to the same tokens/secrets as the [deephaven/deephaven-core](https://github.com/deephaven/deephaven-core) repository, so GitHub actions will fail. To disable GitHub actions in your forked repository, go to "Actions" -> "Disable Actions" in your forked repository settings (`https://github.com//deephaven-core/settings/actions`). Over time, forks will get out of sync with the upstream repository. To stay up to date, either: * Navigate to `https://github.com//deephaven-core` and click on `Fetch upstream`, or From 5682a2c6e7db4a4e1f748b25bd026e478fc924ec Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 21 May 2021 12:47:28 -0500 Subject: [PATCH 02/10] Populate snapshot request in js api, handle optional left table (#648) Fixes #520 --- .../table/ops/SnapshotTableGrpcImpl.java | 26 ++++++++++++++----- .../io/deephaven/web/client/api/JsTable.java | 17 ++++++++++-- .../deephaven/web/client/api/TableTicket.java | 7 +++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/grpc-api/src/main/java/io/deephaven/grpc_api/table/ops/SnapshotTableGrpcImpl.java b/grpc-api/src/main/java/io/deephaven/grpc_api/table/ops/SnapshotTableGrpcImpl.java index a9cb125470c..91db9464d63 100644 --- a/grpc-api/src/main/java/io/deephaven/grpc_api/table/ops/SnapshotTableGrpcImpl.java +++ b/grpc-api/src/main/java/io/deephaven/grpc_api/table/ops/SnapshotTableGrpcImpl.java @@ -2,7 +2,6 @@ import io.deephaven.base.verify.Assert; import io.deephaven.datastructures.util.CollectionUtil; -import com.google.common.collect.Lists; import io.deephaven.db.tables.Table; import io.deephaven.db.tables.live.LiveTableMonitor; import io.deephaven.db.tables.select.SelectColumnFactory; @@ -16,6 +15,8 @@ import javax.inject.Inject; import javax.inject.Singleton; +import java.util.Arrays; +import java.util.Collections; import java.util.List; @Singleton @@ -24,7 +25,12 @@ public class SnapshotTableGrpcImpl extends GrpcTableOperation EXTRACT_DEPS = - (request) -> Lists.newArrayList(request.hasLeftId() ? request.getLeftId() : null, request.getRightId()); + (request) -> { + if (request.hasLeftId()) { + return Arrays.asList(request.getLeftId(), request.getRightId()); + } + return Collections.singletonList(request.getRightId()); + }; @Inject public SnapshotTableGrpcImpl(final LiveTableMonitor liveTableMonitor) { @@ -34,11 +40,17 @@ public SnapshotTableGrpcImpl(final LiveTableMonitor liveTableMonitor) { @Override public Table create(final SnapshotTableRequest request, final List> sourceTables) { - Assert.eq(sourceTables.size(), "sourceTables.size()", 2); - - final SessionState.ExportObject leftSource = sourceTables.get(0); - final Table lhs = leftSource == null ? TableTools.emptyTable(1) : leftSource.get(); - final Table rhs = sourceTables.get(1).get(); + final Table lhs; + final Table rhs; + if (sourceTables.size() == 1) { + lhs = TableTools.emptyTable(1); + rhs = sourceTables.get(0).get(); + } else if (sourceTables.size() == 2) { + lhs = sourceTables.get(0).get(); + rhs = sourceTables.get(1).get(); + } else { + throw Assert.statementNeverExecuted("Unexpected sourceTables size " + sourceTables.size()); + } final String[] stampColumns = request.getStampColumnsList().toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); final SelectColumn[] stampExpressions = SelectColumnFactory.getExpressions(request.getStampColumnsList()); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java index 5dcc321e1e7..b60466130ca 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsTable.java @@ -742,7 +742,13 @@ public Promise treeTable(Object configObject) { @JsMethod public Promise freeze() { return workerConnection.newState((c, state, metadata) -> { - workerConnection.tableServiceClient().snapshot(new SnapshotTableRequest(), metadata, c::apply); + SnapshotTableRequest request = new SnapshotTableRequest(); + request.setLeftid(null);// explicit null to signal that we are just freezing this table + request.setRightid(state().getHandle().makeTableReference()); + request.setResultid(state.getHandle().makeTicket()); + request.setDoinitialsnapshot(true); + request.setStampcolumnsList(new String[0]); + workerConnection.tableServiceClient().snapshot(request, metadata, c::apply); }, "freeze").refetch(this, workerConnection.metadata()).then(state -> Promise.resolve(new JsTable(workerConnection, state))); } @@ -765,7 +771,14 @@ public Promise snapshot(JsTable rightHandSide, @JsOptional Boolean doIn } final String fetchSummary = "snapshot(" + rightHandSide + ", " + doInitialSnapshot + ", " + Arrays.toString(stampColumns) + ")"; return workerConnection.newState((c, state, metadata) -> { - workerConnection.tableServiceClient().snapshot(new SnapshotTableRequest(), metadata, c::apply); + SnapshotTableRequest request = new SnapshotTableRequest(); + request.setLeftid(state().getHandle().makeTableReference()); + request.setRightid(rightHandSide.state().getHandle().makeTableReference()); + request.setResultid(state.getHandle().makeTicket()); + request.setDoinitialsnapshot(realDoInitialSnapshot); + request.setStampcolumnsList(realStampColums); + + workerConnection.tableServiceClient().snapshot(request, metadata, c::apply); }, fetchSummary).refetch(this, workerConnection.metadata()).then(state -> Promise.resolve(new JsTable(workerConnection, state))); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/TableTicket.java b/web/client-api/src/main/java/io/deephaven/web/client/api/TableTicket.java index 3ac526a9c1a..44ab41561df 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/TableTicket.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/TableTicket.java @@ -3,6 +3,7 @@ import elemental2.core.Int32Array; import elemental2.core.Uint8Array; import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.session_pb.Ticket; +import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.table_pb.TableReference; /** * Replacement for TableHandle, wraps up Ticket plus current export state. We only consider the lower bytes for @@ -80,6 +81,12 @@ public Ticket makeTicket() { return ticket; } + public TableReference makeTableReference() { + TableReference reference = new TableReference(); + reference.setTicket(makeTicket()); + return reference; + } + @Override public String toString() { return "TableTicket{" + From 8b7afdfdec98731929091521e57a67f037a7f437 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 21 May 2021 14:02:14 -0500 Subject: [PATCH 03/10] Support for jpy integration junit tests (#632) Partial #430 --- py/jpy-integration/build.gradle | 46 +++++++++++++++++++ .../src/javaToPython/build.gradle.template | 19 ++++++++ .../src/pythonToJava/jpyconfig.py.template | 2 +- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 py/jpy-integration/src/javaToPython/build.gradle.template diff --git a/py/jpy-integration/build.gradle b/py/jpy-integration/build.gradle index 97832b65589..6269880d395 100644 --- a/py/jpy-integration/build.gradle +++ b/py/jpy-integration/build.gradle @@ -7,6 +7,7 @@ plugins { } evaluationDependsOn ':deephaven-jpy' +evaluationDependsOn ':Integrations' sourceSets { // All sourcesets here are used for testing; we'll inform the idea{} plugin, below @@ -110,6 +111,51 @@ Task pythonToJava = tasks.create 'unittestPythonToJava', { } if (!PyEnv.pythonEnabled(project)) { + Closure> gradleTestInDocker = { String taskName, SourceSet sourceSet -> + + def gradleWrapper = tasks.register("${taskName}GradleInit", Wrapper.class) { wrapper -> + wrapper.scriptFile "${buildDir}/template-project/gradlew" + wrapper.jarFile "${buildDir}/template-project/gradle/wrapper/gradle-wrapper.jar" + } + return Docker.registerDockerTask(project, taskName) { + copyIn { + dependsOn gradleWrapper + from ("${buildDir}/template-project") { + into 'project' + } + from(project.file('src/javaToPython/build.gradle.template')) { + into 'project' + rename { file -> 'build.gradle' } + } + from (sourceSet.runtimeClasspath) { + into 'classpath' + } + from (sourceSet.output.getClassesDirs()) { + into 'classes' + } + } + parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + dockerfile { + // base image with default java, python, wheels + from 'deephaven/runtime-base' + + // set up the project + copyFile 'project', '/project' + workingDir '/project' + // run once with no actual classes, so that gradle is preinstalled and cached + runCommand '/project/gradlew' + copyFile 'classpath', '/classpath' + copyFile 'classes', '/classes' + } + entrypoint = ['/project/gradlew', 'test', '--info'] + containerOutPath = '/project/build/test-results/test/' + copyOut { + into "${buildDir}/test-results/${taskName}" + } + } + } + testJavaToPython.dependsOn gradleTestInDocker('java-to-python-test', sourceSets.javaToPython) + // Task testTask = venvTest.javaTest(project, "java-to-python-test-${pv.name}", installWheels, sourceSets.javaToPython) // testJavaToPython.dependsOn testTask diff --git a/py/jpy-integration/src/javaToPython/build.gradle.template b/py/jpy-integration/src/javaToPython/build.gradle.template new file mode 100644 index 00000000000..3025e3821e5 --- /dev/null +++ b/py/jpy-integration/src/javaToPython/build.gradle.template @@ -0,0 +1,19 @@ +plugins { + id 'java-library' +} + +// Classpath is already built, just need to pass to the test task +test.classpath = fileTree('/classpath/').plus(files('/classpath')) +test.testClassesDirs = files('/classes') + +test.systemProperties([ + 'jpy.jpyLib':'/usr/local/lib/python3.7/dist-packages/jpy.cpython-37m-x86_64-linux-gnu.so', + 'jpy.jdlLib':'/usr/local/lib/python3.7/dist-packages/jdl.cpython-37m-x86_64-linux-gnu.so', + 'jpy.pythonLib':'/usr/lib/x86_64-linux-gnu/libpython3.7m.so.1.0', + // Cleaning up on a dedicated thread has some issues when there is frequent starting + // and stopping of the python virtual environment. We'll rely on cleaning up inline + // when necessary. + // TODO issue #651 to see if we can remove this + 'PyObject.cleanup_on_thread':'false', +// 'jpy.debug':'true' +]) diff --git a/py/jpy-integration/src/pythonToJava/jpyconfig.py.template b/py/jpy-integration/src/pythonToJava/jpyconfig.py.template index 3db44b50727..3124f2e1793 100644 --- a/py/jpy-integration/src/pythonToJava/jpyconfig.py.template +++ b/py/jpy-integration/src/pythonToJava/jpyconfig.py.template @@ -2,6 +2,6 @@ java_home = '${javaHome}' jvm_dll = None jvm_maxmem = '${jvmMaxMem}' jvm_classpath = ['${jvmClasspath}'] -jvm_properties = {'PyObject.cleanup_on_thread': 'false'} +jvm_properties = {'PyObject.cleanup_on_thread': 'false'} # TODO issue #651 to see if we can remove this #jvm_options = ['-Djpy.debug=true'] jvm_options = [] From 9db6aaa8635a03aefe14a28ed10d2684fd53d2d4 Mon Sep 17 00:00:00 2001 From: Devin Smith Date: Fri, 21 May 2021 19:28:09 -0500 Subject: [PATCH 04/10] Javadoc search (#645) --- .github/workflows/docs-ci.yml | 21 +- build.gradle | 14 +- buildSrc/build.gradle | 50 ---- buildSrc/settings.gradle | 1 + .../io/deephaven/javadoc/IncludeDoclet.java | 218 -------------- gradle.properties | 3 + javadoc/README.md | 23 ++ javadoc/build.gradle | 20 ++ .../io/deephaven/javadoc/IncludeDoclet.java | 274 ++++++++++++++++++ javadoc/src/main/java/module-info.java | 3 + settings.gradle | 4 + 11 files changed, 356 insertions(+), 275 deletions(-) create mode 100644 buildSrc/settings.gradle delete mode 100644 buildSrc/src/compat/java/io/deephaven/javadoc/IncludeDoclet.java create mode 100644 javadoc/README.md create mode 100644 javadoc/build.gradle create mode 100644 javadoc/src/main/java/io/deephaven/javadoc/IncludeDoclet.java create mode 100644 javadoc/src/main/java/module-info.java diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 1c0b1ea2ae1..407bd9375d5 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -14,26 +14,37 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Setup JDK - id: setup-java + - name: Setup JDK 8 + id: setup-java-8 uses: actions/setup-java@v1 with: java-version: '8.0.292' + - name: Setup JDK 11 + id: setup-java-11 + uses: actions/setup-java@v1 + with: + java-version: '11.0.11' + - name: Setup gradle properties run: | mkdir -p $HOME/.gradle cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties - echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties + echo "org.gradle.java.installations.paths=${{ steps.setup-java-8.outputs.path }},${{ steps.setup-java-11.outputs.path }}" >> $HOME/.gradle/gradle.properties - name: All Javadoc uses: burrunan/gradle-cache-action@v1 with: job-id: allJavadoc - # todo: eventually expand to more - arguments: --scan allJavadoc + arguments: --scan -PincludeJavadoc=true allJavadoc gradle-version: wrapper + - name: Upload Javadocs + uses: actions/upload-artifact@v2 + with: + name: javadocs + path: 'build/docs/javadoc/' + - name: Deploy Javadoc if: ${{ github.ref == 'refs/heads/main' }} uses: burnett01/rsync-deployments@4.1 diff --git a/build.gradle b/build.gradle index c07dc490911..5f0f5018558 100644 --- a/build.gradle +++ b/build.gradle @@ -327,14 +327,24 @@ apply plugin: 'io.deephaven.java-conventions' String javaDocOverviewLocation = 'build/docs/overview.html' tasks.register 'allJavadoc', Javadoc, { Javadoc jdoc -> + jdoc.javadocTool = javaToolchains.javadocToolFor{it -> + languageVersion = JavaLanguageVersion.of(11) + } jdoc.inputs.file javaDocOverviewLocation + if (project.findProperty('includeJavadoc') == 'true') { + jdoc.inputs.files project.project(':javadoc').tasks.getByName('compileJava').outputs.files + } jdoc.options.encoding = 'UTF-8' - jdoc.options.docletpath = [new File("$rootDir/buildSrc/build/classes/java/compat")] + jdoc.options.docletpath = [new File("$rootDir/javadoc/build/classes/java/main")] jdoc.options.doclet = 'io.deephaven.javadoc.IncludeDoclet' jdoc.options.tags = ['Include:a:', 'Exclude:a:', 'IncludeAll:a:', 'apiNote', 'implNote'] // include a note on the front Javadoc page telling us what version the Javadoc was generated from. jdoc.options.overview = new File("$rootDir", javaDocOverviewLocation) - jdoc.options.links = ['https://docs.oracle.com/javase/8/docs/api/'] + jdoc.options.links = ['https://docs.oracle.com/en/java/javase/11/docs/api/'] + jdoc.options.jFlags '--add-exports', 'jdk.internal.opt/jdk.internal.joptsimple=ALL-UNNAMED', + '--add-exports', 'jdk.javadoc/jdk.javadoc.internal.tool=ALL-UNNAMED' + jdoc.options.addStringOption('Xdoclint:none', '-quiet') + jdoc.options.addBooleanOption('-no-module-directories', true) //add to as javadoc gets completed String [] exportedProjects = [ diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index a0892d5c07c..be9ad4aa52c 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -1,6 +1,5 @@ plugins { id 'groovy' - id 'java' id 'idea' id 'groovy-gradle-plugin' } @@ -13,16 +12,11 @@ java { idea { project { - //if you want to set specific jdk and language level jdkName = '1.8' languageLevel = '1.8' } } -sourceSets { - compat { } -} - repositories { mavenCentral() maven { @@ -31,7 +25,6 @@ repositories { } dependencies { - compile('de.esoco.gwt:gwt-gradle-plugin:1.1.1') { exclude group: 'org.codehaus.groovy' } @@ -41,48 +34,5 @@ dependencies { compile('com.netflix.nebula:gradle-ospackage-plugin:8.3.0') compile "gradle.plugin.com.jetbrains.python:gradle-python-envs:0.0.30" - - runtime sourceSets.compat.output } -static def getToolsJar(JavaCompiler compiler) { - File toolsJar = compiler.metadata.installationPath.file('lib/tools.jar').asFile - if (toolsJar.exists()) { - return toolsJar - } - File javaHome = compiler.metadata.installationPath.asFile - if (javaHome.getName().equalsIgnoreCase('jre')) { - javaHome = javaHome.getParentFile() - toolsJar = new File(javaHome, "lib/tools.jar") - if (toolsJar.exists()) { - return toolsJar; - } - } - if (org.gradle.internal.os.OperatingSystem.current().isWindows()) { - String version = compiler.metadata.languageVersion.toString() - if (javaHome.getName().matches("jre\\d+") || javaHome.getName() == "jre${version}") { - javaHome = new File(javaHome.getParentFile(), "jdk${version}") - toolsJar = new File(javaHome, 'lib/tools.jar') - if (toolsJar.exists()) { - return toolsJar - } - } - } - return null -} - -def compiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(8) -} -def toolsJar = getToolsJar(compiler.get()) -if (toolsJar == null) { - throw new IllegalStateException("Unable to find toolsJar for '${compiler.get().metadata.installationPath}'") -} -dependencies.add 'compatCompile', files(toolsJar) - -tasks.named('jar', Jar).configure { - Jar jar -> - jar.preserveFileTimestamps = false - jar.reproducibleFileOrder = true - jar.outputs.cacheIf { true } -} diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle new file mode 100644 index 00000000000..8136419d1ca --- /dev/null +++ b/buildSrc/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'DeephavenCoreBuildsrc' diff --git a/buildSrc/src/compat/java/io/deephaven/javadoc/IncludeDoclet.java b/buildSrc/src/compat/java/io/deephaven/javadoc/IncludeDoclet.java deleted file mode 100644 index 4d461b8ccd8..00000000000 --- a/buildSrc/src/compat/java/io/deephaven/javadoc/IncludeDoclet.java +++ /dev/null @@ -1,218 +0,0 @@ -package io.deephaven.javadoc;// From http://www.sixlegs.com/blog/java/exclude-javadoc-tag.html - -import com.sun.javadoc.*; -import com.sun.tools.doclets.standard.Standard; -import com.sun.tools.javadoc.Main; - -import java.io.FileNotFoundException; -import java.lang.reflect.*; -import java.util.ArrayList; -import java.util.List; - -/** - * Allows for the custom javadoc tags - * Include - * IncludeAll - * Exclude - * With none of these tags, the javadoc will not be included in the produced document. - * - * IncludeAll- add at the beginning of a Class to include all methods, fields, etc. by default. Note inner classes must - * have their own Include tag to be included. - * Include- add to a class, method, or field to include inside the produced javadoc. - * Exclude- add to a class, method, or field to exclude inside the produced javadoc. Only necessary after an IncludeAll tag - */ -public class IncludeDoclet extends com.sun.tools.doclets.standard.Standard { - private static String INCLUDEALL = "IncludeAll"; - private static String INCLUDE = "Include"; - private static String EXCLUDE = "Exclude"; - public static void main(String[] args) throws FileNotFoundException { - debug("main()"); - String name = IncludeDoclet.class.getName(); - Main.execute(name, args); - } - - public static boolean start(RootDoc root) { - debug("start()"); - return Standard.start((RootDoc) process(root, RootDoc.class)); - } - - private static Object process(Object obj, Class expect) { - debug("process(%s, %s)", obj, expect); - if (obj == null) - return null; - Class cls = obj.getClass(); - if (cls.getName().startsWith("com.sun.")) { - return Proxy.newProxyInstance(cls.getClassLoader(), - cls.getInterfaces(), new IncludeHandler(obj)); - } else if (obj instanceof Object[]) { - Class componentType = expect.getComponentType(); - Object[] array = (Object[]) obj; - List list = new ArrayList(array.length); - for (int i = 0; i < array.length; i++) { - Object entry = array[i]; - if ((entry instanceof Doc) && includeAll((Doc) entry)) { - if(entry instanceof ClassDoc) { - list.add(processNeedExclude(entry, componentType)); - continue; - } - } else if ((entry instanceof Doc) && !include((Doc) entry)) { - continue; - } - list.add(process(entry, componentType)); - } - return list.toArray((Object[]) Array.newInstance(componentType, - list.size())); - } else { - return obj; - } - } - - private static Object processNeedExclude(Object obj, Class expect) { - debug("process(%s, %s)", obj, expect); - if (obj == null) - return null; - Class cls = obj.getClass(); - if (cls.getName().startsWith("com.sun.")) { - return Proxy.newProxyInstance(cls.getClassLoader(), - cls.getInterfaces(), new ExcludeHandler(obj)); - } else if (obj instanceof Object[]) { - Class componentType = expect.getComponentType(); - Object[] array = (Object[]) obj; - List list = new ArrayList(array.length); - for (int i = 0; i < array.length; i++) { - Object entry = array[i]; - if ((entry instanceof Doc) && exclude((Doc) entry)) { - continue; - } - list.add(processNeedExclude(entry, componentType)); - } - return list.toArray((Object[]) Array.newInstance(componentType, - list.size())); - } else { - return obj; - } - } - - private static boolean includeAll(Doc doc) { - if (doc instanceof ProgramElementDoc) { - if (((ProgramElementDoc) doc).containingPackage().tags(INCLUDEALL).length > 0) { - debug("includeall(%s) returns true", doc.toString()); - return true; - } - } - boolean result = doc.tags(INCLUDEALL).length > 0; - debug("includeall(%s) returns true", doc.toString()); - return result; - } - - private static boolean include(Doc doc) { - if (doc instanceof ProgramElementDoc) { - if (((ProgramElementDoc) doc).containingPackage().tags(INCLUDE).length > 0) { - debug("include(%s) returns true", doc.toString()); - return true; - } - } - boolean result = doc.tags(INCLUDE).length > 0; - debug("include(%s) returns true", doc.toString()); - return result; - } - - private static boolean exclude(Doc doc) { - if (doc instanceof ProgramElementDoc) { - if (((ProgramElementDoc) doc).containingPackage().tags(EXCLUDE).length > 0) { - debug("exclude(%s) returns true", doc.toString()); - return true; - } - } - //changed to include= maybe change back - boolean result = doc.tags(EXCLUDE).length > 0; - debug("exclude(%s) returns true", doc.toString()); - return result; - } - - private static class IncludeHandler implements InvocationHandler { - private Object target; - - public IncludeHandler(Object target) { - this.target = target; - } - - public Object invoke(Object proxy, Method method, Object[] args) - throws Throwable { - debug("invoke(%s, %s, %s)", "proxy", "method", args); - - if (args != null) { - String methodName = method.getName(); - if (methodName.equals("compareTo") - || methodName.equals("equals") - || methodName.equals("overrides") - || methodName.equals("subclassOf")) { - args[0] = unwrap(args[0]); - } - } - try { - return process(method.invoke(target, args), - method.getReturnType()); - } catch (InvocationTargetException e) { - throw e.getTargetException(); - } - } - - private Object unwrap(Object proxy) { - debug("unwrap(%s)", proxy); - if (proxy instanceof Proxy) { - if(Proxy.getInvocationHandler(proxy) instanceof IncludeHandler) { - return ((IncludeHandler) Proxy.getInvocationHandler(proxy)).target; - } else { - return ((ExcludeHandler) Proxy.getInvocationHandler(proxy)).target; - } - } - return proxy; - } - } - - private static class ExcludeHandler implements InvocationHandler { - private Object target; - - public ExcludeHandler(Object target) { - this.target = target; - } - - public Object invoke(Object proxy, Method method, Object[] args) - throws Throwable { - debug("invokeExcludeHandler(%s, %s, %s)", "proxy", "method", args); - - if (args != null) { - String methodName = method.getName(); - if (methodName.equals("compareTo") - || methodName.equals("equals") - || methodName.equals("overrides") - || methodName.equals("subclassOf")) { - args[0] = unwrap(args[0]); - } - } - try { - return processNeedExclude(method.invoke(target, args), - method.getReturnType()); - } catch (InvocationTargetException e) { - throw e.getTargetException(); - } - } - - private Object unwrap(Object proxy) { - debug("unwrap(%s)", proxy); - if (proxy instanceof Proxy) { - if(Proxy.getInvocationHandler(proxy) instanceof IncludeHandler) { - return ((IncludeHandler) Proxy.getInvocationHandler(proxy)).target; - } else { - return ((ExcludeHandler) Proxy.getInvocationHandler(proxy)).target; - } - } - return proxy; - } - } - - private static void debug(String fmt, Object... o) { - // System.out.printf("IncludeDoclet: " + fmt + "\n", o); - } -} diff --git a/gradle.properties b/gradle.properties index 261ebdf82e3..27b64255995 100644 --- a/gradle.properties +++ b/gradle.properties @@ -71,3 +71,6 @@ yarnWorkDir=web/.cache/yarn nodeModulesIde=web/client-ui/ nodeModulesJupyter=web/jupyter-grid/js/ nodeModulesProto=proto/ + +# See javadoc/README.md for context +includeJavadoc=false diff --git a/javadoc/README.md b/javadoc/README.md new file mode 100644 index 00000000000..aeda2236d09 --- /dev/null +++ b/javadoc/README.md @@ -0,0 +1,23 @@ +# javadoc doclet + +This module builds the doclet that our javadoc building process uses. + +To support [Searchable Javadocs](https://github.com/deephaven/deephaven-core/issues/590) +we need to have a Java 11 doclet and Java 11 javadoc process. + +The majority of developers don't need to build javadocs locally. +In an effort to minimize build dependencies, the javadoc module is disabled by default. + +### Development + +To aid in the doclet development processes, a developer can temporarily set the gradle property +`includeJavadocs` to true in [gradle.properties](../gradle.properties). +This should enable IDEs to pick up the module. + +Alternatively, the setting can be enabled ad-hoc to build the javadocs: + +```shell +./gradlew -PincludeJavadoc=true allJavadoc +``` + +This is essentially what happens in CI, see [docs-ci.yml](../.github/workflows/docs-ci.yml). diff --git a/javadoc/build.gradle b/javadoc/build.gradle new file mode 100644 index 00000000000..032ad9631c4 --- /dev/null +++ b/javadoc/build.gradle @@ -0,0 +1,20 @@ +plugins { + id 'java' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} + +repositories { + mavenCentral() +} + +// Settings these explicitly avoids adding the --release flag, which will blow up w/ --add-exports +compileJava.sourceCompatibility = '11' +compileJava.targetCompatibility = '11' + +compileJava.options.compilerArgs.addAll( + ['--add-exports', 'jdk.javadoc/jdk.javadoc.internal.tool=dh_javadoc']) diff --git a/javadoc/src/main/java/io/deephaven/javadoc/IncludeDoclet.java b/javadoc/src/main/java/io/deephaven/javadoc/IncludeDoclet.java new file mode 100644 index 00000000000..3921ba98fe6 --- /dev/null +++ b/javadoc/src/main/java/io/deephaven/javadoc/IncludeDoclet.java @@ -0,0 +1,274 @@ +package io.deephaven.javadoc; + +import com.sun.source.doctree.BlockTagTree; +import com.sun.source.doctree.DocCommentTree; +import com.sun.source.doctree.DocTree; +import com.sun.source.util.DocTrees; +import jdk.javadoc.doclet.DocletEnvironment; +import jdk.javadoc.doclet.Reporter; +import jdk.javadoc.doclet.StandardDoclet; +import jdk.javadoc.internal.tool.DocEnvImpl; + +import javax.lang.model.SourceVersion; +import javax.lang.model.element.Element; +import javax.lang.model.element.ElementKind; +import javax.lang.model.element.TypeElement; +import javax.lang.model.util.Elements; +import javax.lang.model.util.Types; +import javax.tools.Diagnostic; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import java.util.*; + +/** + * This doclet uses the custom tags "IncludeAll", "Include" and "Exclude" + * to (in part) determine whether javadoc is generated. + * + * IncludeAll- add at the beginning of a Class to include all methods, fields, etc. by default. Note inner classes must + * have their own IncludeAll/Include tags to be included. If an IncludeAll is added to a non-Class, it behaves as an Include. + * Include- add to a class, method, or field to include inside the produced javadoc. + * Exclude- add to a class, method, or field to exclude from the produced javadoc. Only necessary after an IncludeAll tag + */ +public class IncludeDoclet extends StandardDoclet { + private static final String INCLUDEALL = "IncludeAll"; + private static final String INCLUDE = "Include"; + private static final String EXCLUDE = "Exclude"; + private static final boolean debug = false; + private Reporter reporter; + + @Override + public String getName() { + return "IncludeDoclet"; + } + + @Override + public void init(Locale locale, Reporter reporter) { + this.reporter = reporter; + super.init(locale, reporter); + } + + @Override + public boolean run(DocletEnvironment docEnv) { + report("Running IncludeDoclet"); + return super.run(new DocletEnvironmentImplementation(docEnv)); + } + + private void report(final String report) { + if(debug) { + reporter.print(Diagnostic.Kind.NOTE, report); + } + } + + private void reportStartSection() { + if(debug) { + reporter.print(Diagnostic.Kind.NOTE, "----------START SECTION-----------"); + } + } + + private void reportEndSection() { + if(debug) { + reporter.print(Diagnostic.Kind.NOTE, "-----------END SECTION-------------"); + } + } + + /** + * This wrapper extends jdk.javadoc.internal.tool. To run, compile with + * --add-exports jdk.javadoc/jdk.javadoc.internal.tool=ALL-UNNAMED + * + * This will also need to be added to the javadoc command e.g. + * javadoc -J--add-exports -Jjdk.javadoc/jdk.javadoc.internal.tool=ALL-UNNAMED -docletpath ~/dev/Javadoc/src/main/java/ -doclet IncludeDoclet -tag Exclude:a: -tag Include:a: -tag IncludeAll:a: -d ~/dev/Javadoc/build/javadoc com.javadoc > output.txt + */ + private class DocletEnvironmentImplementation extends jdk.javadoc.internal.tool.DocEnvImpl { + + private final Set includeAllClasses = new HashSet<>(); + private final Set notIncludeAllClasses = new HashSet<>(); + + private DocletEnvironmentImplementation(DocletEnvironment docletEnvironment) { + super(((DocEnvImpl) docletEnvironment).toolEnv, ((DocEnvImpl) docletEnvironment).etable); + } + + @Override + public Set getIncludedElements() { + final Set elements = super.getIncludedElements(); + reportStartSection(); + report("Running getIncludedElements"); + report("All elements: " + elements); + report("All element kinds: " + Arrays.toString(elements.stream().map(Element::getKind).toArray())); + + final Set ret = new LinkedHashSet<>(); + + for (final Element e : elements) { + if(isPackageOrModule(e)) { + ret.add(e); + } else { + if(isIncludedCustomTags(e)) { + ret.add(e); + } + } + } + + report("All included elements: " + ret); + report("All included element kinds: " + Arrays.toString(ret.stream().map(Element::getKind).toArray())); + reportEndSection(); + + return ret; + } + + @Override + public boolean isIncluded(Element e) { + reportStartSection(); + final String prefix = "isIncluded: "; + report("Running " + prefix + e); + + boolean include; + if(isPackageOrModule(e)) { + include = super.isIncluded(e); + } else { + include = isIncludedCustomTags(e) && super.isIncluded(e); + } + report(prefix + e.getKind()); + report(prefix + include); + reportEndSection(); + + return include; + } + + @Override + public boolean isSelected(Element e) { + reportStartSection(); + final String prefix = "isSelected: "; + report("Running " + prefix + e); + + boolean select; + if(isPackageOrModule(e)) { + select = super.isSelected(e); + } else { + select = isIncludedCustomTags(e) && super.isSelected(e); + } + report(prefix + e.getKind()); + report(prefix + select); + reportEndSection(); + + return select; + } + + private boolean isIncludedCustomTags(final Element e) { + final DocCommentTree docCommentTree = super.getDocTrees().getDocCommentTree(e); + + if (docCommentTree != null) { + //The custom tags are block tags (as opposed to an inline tag {@example}) + final BlockTagTree[] trees = docCommentTree + .getBlockTags() + .stream() + .filter(bt -> bt.getKind() == DocTree.Kind.UNKNOWN_BLOCK_TAG) + .map(bt -> (BlockTagTree) bt) + .toArray(BlockTagTree[]::new); + + report("Doc trees for element name " + e.getSimpleName() + " of type " + e.getKind()); + report(Arrays.toString(trees)); + + for (BlockTagTree dt : trees) { + final String tagName = dt.getTagName(); + report("TagName " + tagName); + + switch (tagName) { + case INCLUDEALL: + if (isClass(e)) { + includeAllClasses.add(e); + } + return true; + + case INCLUDE: + return true; + + case EXCLUDE: + return false; + } + + } + } + + //nested classes must have their own include tags + return !isClass(e) && hasIncludeAllParent(e); + } + + private boolean isClass(Element e) { + return e != null && e.getKind() == ElementKind.CLASS; + } + + private boolean isIncludeAllClass(final Element e) { + reportStartSection(); + report("Running isIncludeAllClass"); + report("Element " + e); + report("Element kind " + e.getKind()); + + if(!isClass(e)) { + report("Not a class"); + return false; + } else if(isClass(e) && includeAllClasses.contains(e)) { + report("Class found in includeAllClasses"); + reportEndSection(); + return true; + } else if(isClass(e) && notIncludeAllClasses.contains(e)) { + report("Class found in notIncludeAllClasses"); + reportEndSection(); + return false; + } + + + //search doc to find INCLUDEALL tag + final DocCommentTree docCommentTree = super.getDocTrees().getDocCommentTree(e); + + if (docCommentTree != null) { + final String[] trees = docCommentTree + .getBlockTags() + .stream() + .filter(bt -> bt.getKind() == DocTree.Kind.UNKNOWN_BLOCK_TAG) + .map(bt -> ((BlockTagTree) bt).getTagName()) + .filter(tagName -> tagName.equals(INCLUDEALL)) + .toArray(String[]::new); + + report("Looking through block tags " + docCommentTree.getBlockTags()); + report("IncludeAllFound " + Arrays.toString(trees)); + reportEndSection(); + if(trees.length > 0) { + includeAllClasses.add(e); + return true; + } else { + notIncludeAllClasses.add(e); + return false; + } + } + + report("DocCommentTree null"); + reportEndSection(); + return false; + } + + private boolean hasIncludeAllParent(final Element e) { + reportStartSection(); + report("Running hasIncludeAllParent"); + + final Element enclosingElement = e.getEnclosingElement(); + + report("EnclosingElement " + enclosingElement); + report("EnclosingElement kind" + (enclosingElement == null ? "" : enclosingElement.getKind())); + + if(isClass(enclosingElement)) { + final boolean hasIncludeAllParent = isIncludeAllClass(enclosingElement); + report("hasIncludeAllParent=" + hasIncludeAllParent); + reportEndSection(); + return hasIncludeAllParent; + } + + report("hasIncludeAllParent=false"); + reportEndSection(); + return false; + } + + private boolean isPackageOrModule(final Element e) { + return e != null && (e.getKind() == ElementKind.PACKAGE || e.getKind() == ElementKind.MODULE); + } + } +} + diff --git a/javadoc/src/main/java/module-info.java b/javadoc/src/main/java/module-info.java new file mode 100644 index 00000000000..6b4a1a902c4 --- /dev/null +++ b/javadoc/src/main/java/module-info.java @@ -0,0 +1,3 @@ +module dh_javadoc { + requires jdk.javadoc; +} diff --git a/settings.gradle b/settings.gradle index d78903de17d..fcda198bd99 100644 --- a/settings.gradle +++ b/settings.gradle @@ -142,6 +142,10 @@ include(':envoy') include(':grpc-proxy') +if (includeJavadoc == 'true') { + include(':javadoc') +} + // Apply "vanity naming" (look for .gradle files matching ProjectName/ProjectName.gradle) File root = settings.rootDir mods.each { From d23670433d825e303b12eb47ee1153a2909d93e2 Mon Sep 17 00:00:00 2001 From: Cristian Ferretti <37232625+jcferretti@users.noreply.github.com> Date: Sat, 22 May 2021 09:05:06 -0400 Subject: [PATCH 05/10] Fix IllegalStateException in TwoValuesContainer triggered from Index.insert(Chunk). Fixes #652. (#653) * Fix IllegalStateException in TwoValuesContainer triggered from Index.insert(Chunk). Fixes #652. * Add empty check to RspBitmapSequentialBuildder.appendTreeIndexImpl too. Co-authored-by: Cristian Ferretti --- .../v2/utils/RspBitmapSequentialBuilder.java | 3 +++ .../utils/TreeIndexImplSequentialBuilder.java | 3 +++ .../deephaven/db/v2/utils/rsp/RspBitmap.java | 24 ++++++++++--------- .../db/v2/utils/rsp/RspBitmapTest.java | 12 ++++++++++ 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/DB/src/main/java/io/deephaven/db/v2/utils/RspBitmapSequentialBuilder.java b/DB/src/main/java/io/deephaven/db/v2/utils/RspBitmapSequentialBuilder.java index 20428be0b3e..9ab5641c47f 100644 --- a/DB/src/main/java/io/deephaven/db/v2/utils/RspBitmapSequentialBuilder.java +++ b/DB/src/main/java/io/deephaven/db/v2/utils/RspBitmapSequentialBuilder.java @@ -89,6 +89,9 @@ public void appendRange(final long start, final long end) { @Override public void appendTreeIndexImpl(final long shiftAmount, final TreeIndexImpl ix, final boolean acquire) { + if (ix.ixIsEmpty()) { + return; + } if (!(ix instanceof RspBitmap) || rb == null) { ix.ixForEachLongRange((final long start, final long end) -> { appendRange(start + shiftAmount, end + shiftAmount); diff --git a/DB/src/main/java/io/deephaven/db/v2/utils/TreeIndexImplSequentialBuilder.java b/DB/src/main/java/io/deephaven/db/v2/utils/TreeIndexImplSequentialBuilder.java index 4e480ae7b58..0c9c26fd3c1 100644 --- a/DB/src/main/java/io/deephaven/db/v2/utils/TreeIndexImplSequentialBuilder.java +++ b/DB/src/main/java/io/deephaven/db/v2/utils/TreeIndexImplSequentialBuilder.java @@ -71,6 +71,9 @@ public RspBitmap getRspBitmap() { @Override public void appendTreeIndexImpl(final long shiftAmount, final TreeIndexImpl ix, final boolean acquire) { + if (ix.ixIsEmpty()) { + return; + } if (!(ix instanceof RspBitmap) || rb == null) { ix.ixForEachLongRange((final long start, final long end) -> { appendRange(start + shiftAmount, end + shiftAmount); diff --git a/DB/src/main/java/io/deephaven/db/v2/utils/rsp/RspBitmap.java b/DB/src/main/java/io/deephaven/db/v2/utils/rsp/RspBitmap.java index 153ebb167da..ac5c9591e9c 100644 --- a/DB/src/main/java/io/deephaven/db/v2/utils/rsp/RspBitmap.java +++ b/DB/src/main/java/io/deephaven/db/v2/utils/rsp/RspBitmap.java @@ -244,20 +244,22 @@ private Container createOrUpdateContainerForValues(@NotNull final LongChunk key - final short keyLowBits = lowBitsAsShort(key); - final short firstValueLowBits = lowBitsAsShort(firstValue); - return new TwoValuesContainer(keyLowBits, firstValueLowBits); + final short leftLow = lowBitsAsShort(left); + final short rightLow = lowBitsAsShort(right); + return new TwoValuesContainer(leftLow, rightLow); } final short firstValueLowBits = lowBitsAsShort(firstValue); return container.iset(firstValueLowBits); diff --git a/DB/src/test/java/io/deephaven/db/v2/utils/rsp/RspBitmapTest.java b/DB/src/test/java/io/deephaven/db/v2/utils/rsp/RspBitmapTest.java index 958ae561b1c..24f3fb1c6c4 100644 --- a/DB/src/test/java/io/deephaven/db/v2/utils/rsp/RspBitmapTest.java +++ b/DB/src/test/java/io/deephaven/db/v2/utils/rsp/RspBitmapTest.java @@ -4233,4 +4233,16 @@ public void testInvertNoAccForCoverage() { assertEquals(3, result.get(2)); assertEquals(rb.getCardinality() - 1, result.get(picks.getCardinality() - 1)); } + + @Test + public void testAddValuesUnsafeRegression() { + RspBitmap rb = new RspBitmap(); + rb = rb.add(3); + final WritableLongChunk chunk = WritableLongChunk.makeWritableChunk(4); + chunk.setSize(0); + chunk.add(4); + chunk.add(BLOCK_SIZE + 10); + // we saw: "java.lang.IllegalStateException: iv1=3, iv2=4" + rb.addValuesUnsafeNoWriteCheck(chunk, 0, 2); + } } From 5c1da970e31937b36aba13260094c514f1db5004 Mon Sep 17 00:00:00 2001 From: Jianfeng Mao <4297243+jmao-denver@users.noreply.github.com> Date: Mon, 24 May 2021 08:31:30 -0600 Subject: [PATCH 06/10] port the numba integration prototype from Enterprise to OSS, fixes feature-523 (#638) * port the numba integration prototype from Enterprise to OSS, WIP, fixes feature-523 * changes to make python unittest happy * restore important javadoc comments to the Chunk.java files, fixes #523 --- .../db/v2/select/TestConditionFilter.java | 10 +- .../db/v2/select/TestNumbaFormula.java | 278 ----------------- .../io/deephaven/python/ASTHelperTest.java | 57 ---- DB/python/io/deephaven/python/ast_helper.py | 49 --- .../db/v2/select/AbstractConditionFilter.java | 2 +- .../db/v2/select/AbstractFormulaColumn.java | 2 +- .../db/v2/select/ArgumentsBuilder.java | 34 -- .../db/v2/select/ConditionFilter.java | 4 +- .../db/v2/select/DbArrayArgumentsBuilder.java | 69 ---- .../deephaven/db/v2/select/FormulaColumn.java | 2 +- .../db/v2/select/FormulaKernelTypedBase.java | 178 +++++++++++ .../db/v2/select/NumbaCompileTools.java | 295 ------------------ .../db/v2/select/NumbaFormulaColumn.java | 195 ------------ .../v2/select/PrimitiveArgumentsBuilder.java | 49 --- .../PrimitiveArrayArgumentsBuilder.java | 67 ---- .../db/v2/select/PythonVectorFilter.java | 164 ---------- .../db/v2/select/StringArgumentsBuilder.java | 53 ---- .../db/v2/select/python/ArgumentsChunked.java | 124 ++++++++ .../v2/select/python/ArgumentsSingular.java | 141 +++++++++ .../select/python/ConditionFilterPython.java | 74 +++++ .../python/DeephavenCompatibleFunction.java | 79 +++++ .../v2/select/python/FillContextPython.java | 12 + .../FilterKernelPythonChunkedFunction.java | 53 ++++ .../FilterKernelPythonSingularFunction.java | 48 +++ .../v2/select/python/FormulaColumnPython.java | 86 +++++ .../FormulaKernelPythonChunkedFunction.java | 137 ++++++++ .../FormulaKernelPythonSingularFunction.java | 164 ++++++++++ .../db/v2/sources/chunk/BooleanChunk.java | 6 + .../db/v2/sources/chunk/ByteChunk.java | 6 + .../db/v2/sources/chunk/CharChunk.java | 6 + .../deephaven/db/v2/sources/chunk/Chunk.java | 21 ++ .../db/v2/sources/chunk/DoubleChunk.java | 6 + .../db/v2/sources/chunk/FloatChunk.java | 6 + .../db/v2/sources/chunk/IntChunk.java | 6 + .../db/v2/sources/chunk/LongChunk.java | 6 + .../db/v2/sources/chunk/ObjectChunk.java | 6 + .../db/v2/sources/chunk/ShortChunk.java | 6 + .../java/io/deephaven/python/ASTHelper.java | 55 ---- .../io/deephaven/python/ASTHelperSetup.java | 34 -- .../python/deephaven/java/__init__.py | 2 + .../python/deephaven/java/integration.py | 29 ++ .../python/deephaven/java/primitives.py | 16 + .../python/deephaven/numba/__init__.py | 1 + .../python/deephaven/numba/integration.py | 110 +++++++ py/jpy/src/main/java/org/jpy/Assignment.java | 42 +++ py/jpy/src/main/java/org/jpy/PyObject.java | 2 +- .../src/main/java/org/jpy/PyProxyHandler.java | 30 ++ 47 files changed, 1409 insertions(+), 1413 deletions(-) delete mode 100644 DB-test/src/test/java/io/deephaven/db/v2/select/TestNumbaFormula.java delete mode 100644 DB-test/src/test/java/io/deephaven/python/ASTHelperTest.java delete mode 100644 DB/python/io/deephaven/python/ast_helper.py delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/ArgumentsBuilder.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/DbArrayArgumentsBuilder.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/FormulaKernelTypedBase.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/NumbaCompileTools.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/NumbaFormulaColumn.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArgumentsBuilder.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArrayArgumentsBuilder.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/PythonVectorFilter.java delete mode 100644 DB/src/main/java/io/deephaven/db/v2/select/StringArgumentsBuilder.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsChunked.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsSingular.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/ConditionFilterPython.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/DeephavenCompatibleFunction.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/FillContextPython.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonChunkedFunction.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonSingularFunction.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/FormulaColumnPython.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonChunkedFunction.java create mode 100644 DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonSingularFunction.java delete mode 100644 DB/src/main/java/io/deephaven/python/ASTHelper.java delete mode 100644 DB/src/main/java/io/deephaven/python/ASTHelperSetup.java create mode 100644 Integrations/python/deephaven/java/__init__.py create mode 100644 Integrations/python/deephaven/java/integration.py create mode 100644 Integrations/python/deephaven/java/primitives.py create mode 100644 Integrations/python/deephaven/numba/__init__.py create mode 100644 Integrations/python/deephaven/numba/integration.py create mode 100644 py/jpy/src/main/java/org/jpy/Assignment.java diff --git a/DB-test/src/test/java/io/deephaven/db/v2/select/TestConditionFilter.java b/DB-test/src/test/java/io/deephaven/db/v2/select/TestConditionFilter.java index f66f724d0d0..b7212abc7b6 100644 --- a/DB-test/src/test/java/io/deephaven/db/v2/select/TestConditionFilter.java +++ b/DB-test/src/test/java/io/deephaven/db/v2/select/TestConditionFilter.java @@ -280,10 +280,10 @@ public void testComparison() { @Test public void testLoadNumpyTwice() { - Assert.assertNotNull(PyModule.importModule("numba")); + Assert.assertNotNull(PyModule.importModule("deephaven/numba")); Assert.assertNotNull(PyModule.importModule("numpy")); Assert.assertNotNull(PyModule.importModule("deephaven.lang.vectorize_simple")); - Assert.assertNotNull(PyModule.importModule("numba")); + Assert.assertNotNull(PyModule.importModule("deephaven/numba")); Assert.assertNotNull(PyModule.importModule("numpy")); Assert.assertNotNull(PyModule.importModule("deephaven.lang.vectorize_simple")); } @@ -440,10 +440,4 @@ private Index initCheck(String expression, FormulaParserConfiguration parser) { return conditionFilter.filter(testDataTable.getIndex().clone(), testDataTable.getIndex(), testDataTable, false); } - private Index initV3PythonCheck(String expression) { - PythonVectorFilter conditionFilter = new PythonVectorFilter(expression.replaceAll(" = ", " == ")); - conditionFilter.init(testDataTable.getDefinition()); - return conditionFilter.filter(testDataTable.getIndex().clone(), testDataTable.getIndex(), testDataTable, false); - } - } diff --git a/DB-test/src/test/java/io/deephaven/db/v2/select/TestNumbaFormula.java b/DB-test/src/test/java/io/deephaven/db/v2/select/TestNumbaFormula.java deleted file mode 100644 index a56d99c2cff..00000000000 --- a/DB-test/src/test/java/io/deephaven/db/v2/select/TestNumbaFormula.java +++ /dev/null @@ -1,278 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.configuration.Configuration; -import io.deephaven.io.log.LogLevel; -import io.deephaven.io.logger.StreamLoggerImpl; -import io.deephaven.util.process.ProcessEnvironment; -import io.deephaven.compilertools.CompilerTools; -import io.deephaven.db.tables.Table; -import io.deephaven.db.tables.select.QueryScope; -import io.deephaven.db.tables.utils.TableTools; -import io.deephaven.db.v2.TstUtils; -import io.deephaven.jpy.PythonTest; -import org.jpy.PyInputMode; -import org.jpy.PyLib; -import org.jpy.PyObject; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -import static io.deephaven.db.v2.select.FormulaParserConfiguration.dh; -import static io.deephaven.db.v2.select.FormulaParserConfiguration.nb; -import static junit.framework.TestCase.assertEquals; - -@Ignore -public class TestNumbaFormula extends PythonTest { - static { - if (ProcessEnvironment.tryGet() == null) { - ProcessEnvironment.basicInteractiveProcessInitialization(Configuration.getInstance(), TestNumbaFormula.class.getCanonicalName(), new StreamLoggerImpl(System.out, LogLevel.INFO)); - } - } - - private static final boolean ENABLE_COMPILER_TOOLS_LOGGING = Configuration.getInstance().getBooleanForClassWithDefault(TestNumbaFormula.class, "CompilerTools.logEnabled", false); - - private FormulaParserConfiguration prevParserSettings; - - private boolean compilerToolsLogEnabledInitial = false; - - @Before - public void setUp() throws Exception { - compilerToolsLogEnabledInitial = CompilerTools.setLogEnabled(ENABLE_COMPILER_TOOLS_LOGGING); - } - - - @After - public void tearDown() throws Exception { - CompilerTools.setLogEnabled(compilerToolsLogEnabledInitial); - } - - @Before - public void setParser() { - prevParserSettings = FormulaParserConfiguration.parser; - FormulaParserConfiguration.setParser(FormulaParserConfiguration.Numba); - } - - @After - public void restoreParserSettings() { - FormulaParserConfiguration.parser = prevParserSettings; - } - - private static Table getSumTable(int size, String type) { - - return TableTools.emptyTable(size).select( - dh("A = (" + type + ") (k % 17)"), - dh("B = (" + type + ") (k * 11)"), - dh("C = A + B")); - } - - - void testNumericalType(Table t, String columnA, String columnB, String columnSum) { - t = t.where("i % 2 == 0"); - Table sumT = t.update("__testSum__ = " + columnA + " + " + columnB); - assertEquals(sumT.where("(__testSum__) == " + columnSum).size(), t.size()); - sumT = t.update("__testSum__ = " + columnSum + " - " + columnB); - assertEquals(sumT.where("(__testSum__) == " + columnA).size(), t.size()); - sumT = t.updateView("__testSum__ = " + columnSum + " - " + columnB); - assertEquals(sumT.where("(__testSum__) == " + columnA).size(), t.size()); - - sumT = t.update(nb("__testSum__ = " + columnA + " + " + columnB)); - assertEquals(sumT.where(nb("(__testSum__) == " + columnSum)).size(), t.size()); - sumT = t.update(nb("__testSum__ = " + columnSum + " - " + columnB)); - assertEquals(sumT.where(nb("(__testSum__) == " + columnA)).size(), t.size()); - sumT = t.updateView(nb("__testSum__ = " + columnSum + " - " + columnB)); - assertEquals(sumT.where(nb("(__testSum__) == " + columnA)).size(), t.size()); - } - - - static Map javaToPythonCasts = new HashMap<>(); - - { - javaToPythonCasts.put("byte", "int8"); - javaToPythonCasts.put("char", "uint16"); - javaToPythonCasts.put("short", "int16"); - javaToPythonCasts.put("int", "int32"); - javaToPythonCasts.put("long", "uint64"); - javaToPythonCasts.put("float", "float32"); - javaToPythonCasts.put("double", "float64"); - } - - @Test - public void testNumericalTypes() { - String[] types = {"byte", "char", "short", "int", "long", "float", "double"}; - PyObject.executeCode("import numpy as np\n", PyInputMode.SCRIPT); - int[] sizes = {0, 1, 11, 100}; - for (int size : sizes) { - for (String type : types) { - testNumericalType(getSumTable(size, type), "A", "B", "C"); - } - } - } - - private static Table getBoolTable(int size) { - return TableTools.emptyTable(size).select(dh("A = ((k%3) == 0)"), dh("B = ((k%2) == 0)")); - } - - - @Test - public void testOneValue() { - TstUtils.assertTableEquals(TableTools.emptyTable(1).update("Testv = 1"), TableTools.emptyTable(1).update(dh("Testv = 1"))); - } - - @Test - public void testScope() { - PyObject.executeCode("foo = 1", PyInputMode.STATEMENT); - //Check that the evaluation completes at all - TableTools.emptyTable(1).update("Testv = foo + 1"); - //... and then does it again to confirm it does the right thing - TstUtils.assertTableEquals(TableTools.emptyTable(1).update("Testv = foo + 1"), TableTools.emptyTable(1).update(dh("Testv =(long) 2"))); - - PyObject.executeCode("def testFoo2(incoming):\n" + - " foo = 11\n" + - " return incoming.update('Testv = foo + 1')\n", PyInputMode.SCRIPT); - PyLib.getMainGlobals().asDict().setItem("emptyOne", TableTools.emptyTable(1)); - PyObject pyObject = PyObject.executeCode("testFoo2(emptyOne)", PyInputMode.EXPRESSION); - TstUtils.assertTableEquals( - pyObject.createProxy(Table.class), - TableTools.emptyTable(1).update(dh("Testv =(long) 12"))); - - TstUtils.assertTableEquals(TableTools.emptyTable(1).update("Testv = foo + 1"), TableTools.emptyTable(1).update(dh("Testv = (long)2"))); - - PyObject.executeCode("def testFoo3(incoming,foo):\n" + - " return incoming.update('Testv = foo + 1')\n", PyInputMode.SCRIPT); - pyObject = PyObject.executeCode("testFoo3(emptyOne,10)", PyInputMode.EXPRESSION); - TstUtils.assertTableEquals( - pyObject.createProxy(Table.class), - TableTools.emptyTable(1).update(dh("Testv = (long)11"))); - - } - - @Test - public void testBoolean() { - int sizes[] = {0, 1, 11, 100}; - for (int size : sizes) { - Table t = getBoolTable(size); - Table allTs = t.update("Odds = not B", "Even = True & B", "BySix = B and A", "By8 = (k % 8) == 0", "TrivialT = True", "TrivialF = False"); - FormulaParserConfiguration setParser = FormulaParserConfiguration.parser; - TstUtils.assertTableEquals(allTs.where("Odds").view("A", "B"), t.where("k % 2 == 1")); - TstUtils.assertTableEquals(allTs.where("Even").view("A", "B"), t.where("k % 2 == 0")); - TstUtils.assertTableEquals(allTs.where("BySix").view("A", "B"), t.where("k % 6 == 0")); - TstUtils.assertTableEquals(allTs.where("By8").view("A", "B"), t.where("k % 8 == 0")); - TstUtils.assertTableEquals(allTs.where("Odds").view("A", "B"), t.where(dh("k % 2 == 1"))); - TstUtils.assertTableEquals(allTs.where("Even").view("A", "B"), t.where(dh("k % 2 == 0"))); - TstUtils.assertTableEquals(allTs.where("BySix").view("A", "B"), t.where(dh("k % 6 == 0"))); - TstUtils.assertTableEquals(allTs.where("By8").view("A", "B"), t.where(dh("k % 8 == 0"))); - TstUtils.assertTableEquals(allTs.where("TrivialT").view("A", "B"), t.where(dh("true"))); - TstUtils.assertTableEquals(allTs.where("TrivialF").view("A", "B"), t.where(dh("false"))); - } - } - - private static Table getStringTable(int size) { - return TableTools.emptyTable(size).select(dh("A = `` + (k * 11)")); - } - - - @Test - public void testString() { - if (NumbaCompileTools.pythonVersion.startsWith("2.")) { - return; - } - int sizes[] = {0, 1, 11, 100}; - for (int size : sizes) { - Table t = getStringTable(size); - Table allTs = t.update("B = A + 'works'", "C = B[1]=='5'"); - TstUtils.assertTableEquals(allTs, t.update(dh("B = A + `works`"), dh("C = B.charAt(1)=='5'"))); - Table tt = allTs.where("B[1]=='5'"); - TstUtils.assertTableEquals(tt, allTs.where(dh("B.charAt(1)=='5'"))); - } - - for (int size : sizes) { - Table t = TableTools.emptyTable(size); - t = t.select("A = 'something '"); - Table allTs = t.update("B = A + 'works'", "C = B[1]=='5'"); - TstUtils.assertTableEquals(allTs, t.update(dh("B = A + `works`"), dh("C = B.charAt(1)=='5'"))); - Table tt = allTs.where("B[1]=='5'"); - TstUtils.assertTableEquals(tt, allTs.where(dh("B.charAt(1)=='5'"))); - } - - PyObject.executeCode("foo = 'something '", PyInputMode.STATEMENT); - for (int size : sizes) { - Table t = TableTools.emptyTable(size); - t = t.select("A = foo + foo"); - Table allTs = t.update("B = A + 'works'", "C = B[1]=='5'"); - TstUtils.assertTableEquals(allTs, t.update(dh("B = A + `works`"), dh("C = B.charAt(1)=='5'"))); - Table tt = allTs.where("B[1]=='5'"); - TstUtils.assertTableEquals(tt, allTs.where(dh("B.charAt(1)=='5'"))); - } - - } - - private static Table getArrayTable(int size) { - QueryScope.getDefaultInstance().putParam("size", size); - return TableTools.emptyTable(size).select(dh("A7 = k%7"), dh("A13 = k%13"), dh("halfSize = k % size")); - } - - @Test - public void testUint32Type() { - Table t = TableTools.emptyTable(1); - PyObject.executeCode("import numpy", PyInputMode.SCRIPT); - PyObject.executeCode("my32Int = numpy.uint32(15)", PyInputMode.SCRIPT); - Table tn = t.update(nb("newCol=my32Int")); - TstUtils.assertTableEquals(tn, t.update(dh("newCol = (long)15"))); - } - - - @Test - public void testArraysEtc() { - PyObject.executeCode("import numpy as foo_np", PyInputMode.STATEMENT); - PyObject.executeCode("mult = 10", PyInputMode.STATEMENT); - int sizes[] = {0, 1, 11, 100}; - for (int size : sizes) { - QueryScope.getDefaultInstance().putParam("size", size); - Table source = TableTools.emptyTable(size).select(dh("A7 = k%7"), dh("A13 = k%13"), dh("halfSize = k % (size/2)")); - String aggregationCriteria[] = {"x = i", "x = k % 5", "x = 1", "x = k % 10"}; - for (String aggregationCriterion : aggregationCriteria) { - Table referenceTable = source.by(aggregationCriterion); - Table inputs[] = { - referenceTable, - referenceTable.update(dh("A7 = A7.toArray()"), dh("A13 = A13.toArray()"), dh("halfSize = halfSize.toArray()")) - }; - Table expected = referenceTable.update(dh("med13 = avg(A13)"), dh("avgAll = sum(halfSize)"), dh("prod13 = A13*10"), dh("first = A13.subArray(0,1)")); - Table expectedResults[] = { - expected.update(dh("first = first.toArray()")), - expected.update(dh("A7 = A7.toArray()"), dh("A13 = A13.toArray()"), dh("halfSize = halfSize.toArray()"),dh("first = first.toArray()")) - }; - for (int i = 0; i < inputs.length; i++) { - Table input = inputs[i]; - - //TableTools.show(input.update("med13 = foo_np.mean(A13)", "avgAll = foo_np.sum(halfSize)", "prod13 = A13*mult")); - TstUtils.assertTableEquals(input.update("med13 = foo_np.mean(A13)", "avgAll = foo_np.sum(halfSize)", "prod13 = A13*mult", "first = A13[0:1]"), - expectedResults[i]); - } - } - } - } - - private static class TypeThatDoesNotExistInPython { } - - /** - * IDS-6063 - */ - @Test - public void testNumbaFormulaNotInfluencedByUnusedColumns() { - QueryScope.getDefaultInstance().putParam("java_only_object", new TypeThatDoesNotExistInPython()); - - Table expected = TableTools.emptyTable(1) - .update(dh("JavaOnly=java_only_object")) - .update(dh("Foo=`bar`")); - - Table actual = TableTools.emptyTable(1) - .update(dh("JavaOnly=java_only_object")) - .update(nb("Foo='bar'")); - - TstUtils.assertTableEquals(expected, actual); - } -} diff --git a/DB-test/src/test/java/io/deephaven/python/ASTHelperTest.java b/DB-test/src/test/java/io/deephaven/python/ASTHelperTest.java deleted file mode 100644 index ae6b8de2373..00000000000 --- a/DB-test/src/test/java/io/deephaven/python/ASTHelperTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package io.deephaven.python; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.deephaven.jpy.PythonTest; -import java.util.List; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class ASTHelperTest extends PythonTest { - - private ASTHelper ast; - - @Before - public void setUp() { - ast = ASTHelperSetup.create(); - } - - @After - public void tearDown() { - ast.close(); - } - - @Test - public void extract_expression_names() { - check("f(x[y]) + 2 - A * B", "A", "B", "f", "x", "y"); - check("FOO * BAR", "BAR", "FOO"); - check("BAR", "BAR"); - check("not X", "X"); - check("1"); - // note: while this *may* look like it should fail, we shouldn't dictate at this level what - // constitutes a valid RHS. It's possible that we might want to map python tuple types into - // a common java format. For now, we'll let it get thrown to the real formula parser to error - // out if necessary - check("(foo,bar)", "bar", "foo"); - check("None"); - check("True"); - check("False"); - } - - @Test(expected = Exception.class) - public void not_an_expression() { - ast.extract_expression_names("not_an_expression=1"); - } - - @Test(expected = Exception.class) - public void junk() { - ast.extract_expression_names("-junk-"); - } - - private void check(String formula, String... expects) { - final List results = ASTHelper - .convertToList(ast.extract_expression_names(formula)); - assertThat(results).containsExactly(expects); - } -} diff --git a/DB/python/io/deephaven/python/ast_helper.py b/DB/python/io/deephaven/python/ast_helper.py deleted file mode 100644 index 7f2e8534b9b..00000000000 --- a/DB/python/io/deephaven/python/ast_helper.py +++ /dev/null @@ -1,49 +0,0 @@ -import ast - -class EnsureIsExpression(ast.NodeVisitor): - """ - Ensure the node is an Expression node - """ - def __init__(self): - self.expression = None - - def visit_Expression(self, node): - self.expression = node - - def generic_visit(self, node): - raise Exception('Expecting an AST Expression node, is instead {}'.format(node)) - -class ExtractNamesVisitor(ast.NodeVisitor): - """ - Extract all Name node ids recursively - """ - - def __init__(self): - self.names = set() - - def visit_Name(self, node): - # In Python 3, these nodes are of type NameConstant, and not Name, so they - # won't be included. With Python 2 though, it's best to explicitly exclude - # them. - if not node.id in ["True", "False", "None"]: - self.names.add(node.id) - self.generic_visit(node) - -def extract_expression_names(expression_str): - """ - Extract names from an expression string - - :param expression_str: the python expression, of the form - :return: a sorted list of the named elements from - """ - node = ast.parse(expression_str, mode='eval') - - expression_visitor = EnsureIsExpression() - expression_visitor.visit(node) - - # f(x[y]) + 2 - A * B - names_visitor = ExtractNamesVisitor() - names_visitor.visit(expression_visitor.expression) - - # [A, B, f, x, y] - return sorted(list(names_visitor.names)) diff --git a/DB/src/main/java/io/deephaven/db/v2/select/AbstractConditionFilter.java b/DB/src/main/java/io/deephaven/db/v2/select/AbstractConditionFilter.java index 8c3ebc87e1b..d01ba55572c 100644 --- a/DB/src/main/java/io/deephaven/db/v2/select/AbstractConditionFilter.java +++ b/DB/src/main/java/io/deephaven/db/v2/select/AbstractConditionFilter.java @@ -43,7 +43,7 @@ protected AbstractConditionFilter(@NotNull String formula, boolean unboxArgument this.innerToOuterNames = Collections.emptyMap(); } - AbstractConditionFilter(@NotNull String formula, Map renames, boolean unboxArguments) { + protected AbstractConditionFilter(@NotNull String formula, Map renames, boolean unboxArguments) { this.formula = formula; this.outerToInnerNames = renames; this.unboxArguments = unboxArguments; diff --git a/DB/src/main/java/io/deephaven/db/v2/select/AbstractFormulaColumn.java b/DB/src/main/java/io/deephaven/db/v2/select/AbstractFormulaColumn.java index 82efd96a234..db99dfa9b0d 100644 --- a/DB/src/main/java/io/deephaven/db/v2/select/AbstractFormulaColumn.java +++ b/DB/src/main/java/io/deephaven/db/v2/select/AbstractFormulaColumn.java @@ -65,7 +65,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { * @param formulaString the formula string to be parsed by the DBLanguageParser * @param useKernelFormulas */ - AbstractFormulaColumn(String columnName, String formulaString, boolean useKernelFormulas) { + protected AbstractFormulaColumn(String columnName, String formulaString, boolean useKernelFormulas) { this.formulaString = Require.neqNull(formulaString, "formulaString"); this.columnName = DBNameValidator.validateColumnName(columnName); this.useKernelFormulas = useKernelFormulas; diff --git a/DB/src/main/java/io/deephaven/db/v2/select/ArgumentsBuilder.java b/DB/src/main/java/io/deephaven/db/v2/select/ArgumentsBuilder.java deleted file mode 100644 index 7885853288f..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/ArgumentsBuilder.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.base.Pair; -import io.deephaven.db.tables.dbarrays.DbArrayBase; -import io.deephaven.db.v2.sources.chunk.Chunk; - -import java.util.concurrent.ConcurrentHashMap; - -public interface ArgumentsBuilder { - static ConcurrentHashMap,ArgumentsBuilder> builderCache = new ConcurrentHashMap<>(); - - static ArgumentsBuilder getBuilder(Class usedColumnsType, String npTypeName) { - return builderCache.computeIfAbsent(new Pair<>(usedColumnsType,npTypeName),k -> { - if (usedColumnsType.isPrimitive() || usedColumnsType == Boolean.class) { - return new PrimitiveArgumentsBuilder(npTypeName,usedColumnsType); - } else if (usedColumnsType.isArray() && usedColumnsType.getComponentType().isPrimitive()) { - return new PrimitiveArrayArgumentsBuilder(npTypeName); - } else if (usedColumnsType == String.class) { - return new StringArgumentsBuilder(); - } else if (DbArrayBase.class.isAssignableFrom(usedColumnsType)) { - return new DbArrayArgumentsBuilder(npTypeName); - } - throw new RuntimeException("Arguments of type " + usedColumnsType + " not supported"); - }); - } - - interface Context {} - - int getArgumentsCount(); - Object[] packArguments(Chunk incomingData, CONTEXT context, int size); - - CONTEXT getContext(); - -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/ConditionFilter.java b/DB/src/main/java/io/deephaven/db/v2/select/ConditionFilter.java index 8a0200bfd00..b431085ef8f 100644 --- a/DB/src/main/java/io/deephaven/db/v2/select/ConditionFilter.java +++ b/DB/src/main/java/io/deephaven/db/v2/select/ConditionFilter.java @@ -61,7 +61,7 @@ public static SelectFilter createConditionFilter(@NotNull String formula, Formul case Deephaven: return new ConditionFilter(formula); case Numba: - return new PythonVectorFilter(formula); + throw new UnsupportedOperationException("Python condition filter should be created from python"); default: throw new UnsupportedOperationException("Unknow parser type " + parser); } @@ -266,7 +266,7 @@ public static class ChunkFilter implements Filter { private final String[] columnNames; private final int chunkSize; - ChunkFilter(FilterKernel filterKernel, String[] columnNames, int chunkSize) { + public ChunkFilter(FilterKernel filterKernel, String[] columnNames, int chunkSize) { this.filterKernel = filterKernel; this.columnNames = columnNames; this.chunkSize = chunkSize; diff --git a/DB/src/main/java/io/deephaven/db/v2/select/DbArrayArgumentsBuilder.java b/DB/src/main/java/io/deephaven/db/v2/select/DbArrayArgumentsBuilder.java deleted file mode 100644 index 5d0658a975e..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/DbArrayArgumentsBuilder.java +++ /dev/null @@ -1,69 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.tables.dbarrays.DbArrayBase; -import io.deephaven.db.v2.sources.chunk.*; -import org.jpy.PyModule; -import org.jpy.PyObject; - -import java.lang.reflect.Array; - -public class DbArrayArgumentsBuilder implements ArgumentsBuilder { - - private final Class javaType; - private final String npElementTypeName; - - class Context implements ArgumentsBuilder.Context { - - private final PyObject npInt32; - private final PyModule np; - private final Object npType; - - int[] tentativeOffsetsDestination; - Context(){ - this.np = PyModule.importModule("numpy"); - this.npType = np.getAttribute(npElementTypeName); - this.npInt32 = np.getAttribute("int32"); - } - } - - public DbArrayArgumentsBuilder(String npTypeName) { - npElementTypeName = npTypeName.substring(0, npTypeName.length() - 3); - this.javaType = NumbaCompileTools.toJavaType(npElementTypeName); - } - - - @Override - public int getArgumentsCount() { - return 2; - } - - @Override - public Object[] packArguments(Chunk incomingData, Context context, int size) { - int[] offsetDest = context.tentativeOffsetsDestination != null && context.tentativeOffsetsDestination.length == size ? - context.tentativeOffsetsDestination : new int[size]; - context.tentativeOffsetsDestination = offsetDest; - int totalSize = 0; - ObjectChunk typedChunk = incomingData.asObjectChunk(); - for (int i = 0; i < size; i++) { - totalSize += typedChunk.get(i).size(); - - } - Object destArray = Array.newInstance(javaType, totalSize); - WritableChunk chunkDest = ChunkType.fromElementType(javaType).writableChunkWrap(destArray, 0, totalSize); - int offset = 0; - for (int i = 0; i < size; i++) { - DbArrayBase srcArray = typedChunk.get(i); - int len = srcArray.intSize(); - srcArray.fillChunk(chunkDest.slice(offset,len)); - offset += len; - offsetDest[i] = offset; - } - - return new Object[]{context.np.call("asarray", destArray, context.npType),context.np.call("asarray", offsetDest, context.npInt32)}; - } - - @Override - public Context getContext() { - return new Context(); - } -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/FormulaColumn.java b/DB/src/main/java/io/deephaven/db/v2/select/FormulaColumn.java index 9a4baf62257..fb16f18e133 100644 --- a/DB/src/main/java/io/deephaven/db/v2/select/FormulaColumn.java +++ b/DB/src/main/java/io/deephaven/db/v2/select/FormulaColumn.java @@ -10,7 +10,7 @@ static FormulaColumn createFormulaColumn(String columnName, String formulaString case Deephaven: return new DhFormulaColumn(columnName, formulaString); case Numba: - return new NumbaFormulaColumn(columnName, formulaString); + throw new UnsupportedOperationException("Python formula columns must be created from python"); default: throw new UnsupportedOperationException("Parser support not implemented for " + parser); } diff --git a/DB/src/main/java/io/deephaven/db/v2/select/FormulaKernelTypedBase.java b/DB/src/main/java/io/deephaven/db/v2/select/FormulaKernelTypedBase.java new file mode 100644 index 00000000000..fb899e5e6df --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/FormulaKernelTypedBase.java @@ -0,0 +1,178 @@ +package io.deephaven.db.v2.select; + +import io.deephaven.db.v2.select.Formula.FillContext; +import io.deephaven.db.v2.select.formula.FormulaKernel; +import io.deephaven.db.v2.sources.chunk.Attributes.Any; +import io.deephaven.db.v2.sources.chunk.Attributes.Values; +import io.deephaven.db.v2.sources.chunk.BooleanChunk; +import io.deephaven.db.v2.sources.chunk.ByteChunk; +import io.deephaven.db.v2.sources.chunk.CharChunk; +import io.deephaven.db.v2.sources.chunk.Chunk; +import io.deephaven.db.v2.sources.chunk.Chunk.Visitor; +import io.deephaven.db.v2.sources.chunk.DoubleChunk; +import io.deephaven.db.v2.sources.chunk.FloatChunk; +import io.deephaven.db.v2.sources.chunk.IntChunk; +import io.deephaven.db.v2.sources.chunk.LongChunk; +import io.deephaven.db.v2.sources.chunk.ObjectChunk; +import io.deephaven.db.v2.sources.chunk.ShortChunk; +import io.deephaven.db.v2.sources.chunk.WritableBooleanChunk; +import io.deephaven.db.v2.sources.chunk.WritableByteChunk; +import io.deephaven.db.v2.sources.chunk.WritableCharChunk; +import io.deephaven.db.v2.sources.chunk.WritableChunk; +import io.deephaven.db.v2.sources.chunk.WritableDoubleChunk; +import io.deephaven.db.v2.sources.chunk.WritableFloatChunk; +import io.deephaven.db.v2.sources.chunk.WritableIntChunk; +import io.deephaven.db.v2.sources.chunk.WritableLongChunk; +import io.deephaven.db.v2.sources.chunk.WritableObjectChunk; +import io.deephaven.db.v2.sources.chunk.WritableShortChunk; + +/** + * Extends {@link FormulaKernel} for specifically typed destination + * {@link WritableChunk WritableChunks}. + */ +public abstract class FormulaKernelTypedBase implements FormulaKernel { + + @Override + public final void applyFormulaChunk( + FillContext __context, + WritableChunk __destination, + Chunk[] __sources) { + __destination.walk(new ToTypedMethod<>(__context, __sources)); + } + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableByteChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableBooleanChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableCharChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableShortChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableIntChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableLongChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableFloatChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableDoubleChunk __destination, + Chunk[] __sources); + + public abstract void applyFormulaChunk( + Formula.FillContext __context, + WritableObjectChunk __destination, + Chunk[] __sources); + + private class ToTypedMethod implements Visitor { + private final FillContext __context; + private final Chunk[] __sources; + + ToTypedMethod(FillContext __context, Chunk[] __sources) { + this.__context = __context; + this.__sources = __sources; + } + + @Override + public void visit(ByteChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableByteChunk(), + __sources); + } + + @Override + public void visit(BooleanChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableBooleanChunk(), + __sources); + } + + @Override + public void visit(CharChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableCharChunk(), + __sources); + } + + @Override + public void visit(ShortChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableShortChunk(), + __sources); + } + + @Override + public void visit(IntChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableIntChunk(), + __sources); + } + + @Override + public void visit(LongChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableLongChunk(), + __sources); + } + + @Override + public void visit(FloatChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableFloatChunk(), + __sources); + } + + @Override + public void visit(DoubleChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableDoubleChunk(), + __sources); + } + + @Override + public void visit(ObjectChunk chunk) { + //noinspection unchecked,rawtypes + applyFormulaChunk( + __context, + ((WritableChunk)chunk).asWritableObjectChunk(), + __sources); + } + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/NumbaCompileTools.java b/DB/src/main/java/io/deephaven/db/v2/select/NumbaCompileTools.java deleted file mode 100644 index 61bcf1ef3d3..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/NumbaCompileTools.java +++ /dev/null @@ -1,295 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.tables.ColumnDefinition; -import io.deephaven.db.tables.TableDefinition; -import io.deephaven.db.tables.dbarrays.DbArrayBase; -import io.deephaven.db.v2.sources.chunk.Attributes; -import io.deephaven.db.v2.sources.chunk.Chunk; -import io.deephaven.python.ASTHelper; -import io.deephaven.python.ASTHelperSetup; -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.Set; -import org.jpy.PyDictWrapper; -import org.jpy.PyInputMode; -import org.jpy.PyLib; -import org.jpy.PyObject; - -import java.lang.reflect.Array; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -public class NumbaCompileTools { - - public static final String NUMPY_ARRAY_PREFIX = "array("; - public static final String NUMPY_ARRAY_SUFFIX_C = ", 1d, C)"; - public static final String NUMPY_ARRAY_SUFFIX_A = ", 1d, A)"; - - static class NumbaEvaluator { - private final PyObject vectorized; - private final Class[] usedColumnsTypes; - private final String[] columnsNpTypeName; - - NumbaEvaluator(PyObject vectorized, Class[] usedColumnsTypes, String[] columnsNpTypeName) { - this.vectorized = vectorized; - this.usedColumnsTypes = usedColumnsTypes; - this.columnsNpTypeName = columnsNpTypeName; - } - - - class Context { - - private final Object[] args; - private final ArgumentsBuilder argumentsBuilders[]; - private final ArgumentsBuilder.Context subContexts[]; - - - public Context() { - int argumentsCount = 1; - argumentsBuilders = new ArgumentsBuilder[usedColumnsTypes.length]; - subContexts = new ArgumentsBuilder.Context[usedColumnsTypes.length]; - for (int i = 0; i < usedColumnsTypes.length; i++) { - argumentsBuilders[i] = ArgumentsBuilder.getBuilder(usedColumnsTypes[i], columnsNpTypeName[i]); - argumentsCount += argumentsBuilders[i].getArgumentsCount(); - subContexts[i] = argumentsBuilders[i].getContext(); - } - args = new Object[argumentsCount]; - } - - } - - - public PyObject numbaEvaluate(Context context, Chunk[] inputChunks, int size) { - Object[] args = context.args; - args[0] = size; - int offset = 1; - for (int i = 0; i < context.argumentsBuilders.length; i++) { - Object[] individualArgs = context.argumentsBuilders[i].packArguments(inputChunks[i], context.subContexts[i], size); - System.arraycopy(individualArgs, 0, args, offset, individualArgs.length); - offset += individualArgs.length; - } - - return vectorized.call("__call__", new Object[]{args}); - } - - public Context getContext() { - return new Context(); - } - } - - static PyDictWrapper compileFormula(String formula, Map columnNameMap) { - final Set formulaNames; - try { - formulaNames = new HashSet<>(ASTHelper.convertToList( - ASTHelperSetup.getInstance().extract_expression_names(formula))); - } catch (Exception e) { - throw new FormulaCompilationException("Invalid formula:\n" + formula, e); - } - - // IDS-6063 - // Ensure we don't pass columns that the expression doesn't even reference - - final String[] columnName = columnNameMap.entrySet() - .stream() - .filter(e -> formulaNames.contains(e.getKey())) - .map(Entry::getKey) - .toArray(String[]::new); - - final String[] columnTypePython = columnNameMap.entrySet() - .stream() - .filter(e -> formulaNames.contains(e.getKey())) - .map(e -> toPyType(e.getValue().getDataType(), e.getValue().getComponentType(), true)) - .toArray(String[]::new); - - // IDS-6073 - // todo: clean this up! :O - exec("import numpy\nimport deephaven.lang.vectorize_simple as __vectorize_simple__\n"); - try (final PyDictWrapper effectiveGlobals = getEffectiveGlobals().asDict()) { - effectiveGlobals.setItem("__column_names__", columnName); - effectiveGlobals.setItem("__column_types__", columnTypePython); - effectiveGlobals.setItem("__internal_formula__", formula); - } - // hokay! ready to compile vectorized function. This will return a map like so: - // {fun: vec_function, columns: used_cols, references: used_refs } - try { - return eval("__vectorize_simple__.compile_function(__internal_formula__,zip(__column_names__,__column_types__),__vectorize_simple__.export_locals(globals(),locals()))").asDict(); - // vec.callMethod("compile_function", formula, allCols.unwrap()).asDict(); - } catch (Exception e) { - throw new FormulaCompilationException("Invalid formula:\n" + formula, e); - } - } - - private static PyObject getEffectiveGlobals() { - PyObject currentGlobals = PyLib.getCurrentGlobals(); - if (currentGlobals == null) { - return PyLib.getMainGlobals(); - } - return currentGlobals; - } - - private static PyObject eval(String formula) { - final PyObject currentGlobals = PyLib.getCurrentGlobals(); - if (currentGlobals == null) { - return PyObject.executeCode(formula, PyInputMode.EXPRESSION); - } - try (final PyObject currentLocals = PyLib.getCurrentLocals()) { - return PyObject.executeCode(formula, PyInputMode.EXPRESSION, currentGlobals, currentLocals); - } finally { - currentGlobals.close(); - } - - } - - private static void exec(String formula) { - final PyObject currentGlobals = PyLib.getCurrentGlobals(); - if (currentGlobals == null) { - //noinspection EmptyTryBlock - try (final PyObject obj = PyObject.executeCode(formula, PyInputMode.SCRIPT)) { - - } - return; - } - //noinspection EmptyTryBlock - try ( - final PyObject currentLocals = PyLib.getCurrentLocals(); - final PyObject obj = PyObject.executeCode(formula, PyInputMode.SCRIPT, currentGlobals, currentLocals)) { - - } finally { - currentGlobals.close(); - } - } - - /** - * Returns a numba recognizable type string corresponding to a DH cell type - * - * @param type The main cell type - * @param componentType The component type in case the main type is a DbArray - * @param noFail Indicate whether the method is supposed to throw an exception of silently return null in case of mismatch - * @return - */ - public static String toPyType(Class type, Class componentType, boolean noFail) { - String suffix = ""; - if (type.isArray()) { - suffix = "[:"; - type = type.getComponentType(); - while (type.isArray()) { - suffix += ",:"; - type = type.getComponentType(); - - } - suffix += "]"; - - } - String result; - if (type == String.class) { - result = "unicode_type"; - } else if (type == double.class || type == Double.class) { - result = "float64"; - } else if (type == float.class || type == Float.class) { - result = "float32"; - } else if (type == long.class || type == Long.class) { - result = "int64"; - } else if (type == int.class || type == Integer.class) { - result = "int32"; - } else if (type == short.class || type == Short.class) { - result = "int16"; - } else if (type == char.class || type == Character.class) { - result = "uint16"; - } else if (type == byte.class || type == Byte.class) { - result = "int8"; - } else if (type == boolean.class || type == Boolean.class) { - result = "bool_"; - } else if (type.isPrimitive()) { - if (noFail) { - result = "object"; - } - throw new UnsupportedOperationException(type + " not (yet) supported"); - } else if (DbArrayBase.class.isAssignableFrom(type) && componentType != null) { - return toPyType(componentType, null, noFail) + "[:]"; - } else { - if (noFail) { - result = "object"; - } - throw new UnsupportedOperationException(type + " not supported"); - } - return result + suffix; - } - - static final Map pythonToJavaType; - - public static final String pythonVersion; - - static { - Map pythonToJavaTypeTemp = new HashMap<>(); - pythonToJavaTypeTemp.put("float64", double.class); - pythonToJavaTypeTemp.put("float_", double.class); - pythonToJavaTypeTemp.put("float32", float.class); - pythonToJavaTypeTemp.put("int64", long.class); - pythonToJavaTypeTemp.put("int_", long.class); - pythonToJavaTypeTemp.put("intc", long.class); - pythonToJavaTypeTemp.put("intp", long.class); - pythonToJavaTypeTemp.put("uint64", long.class); - pythonToJavaTypeTemp.put("int32", int.class); - pythonToJavaTypeTemp.put("uint32", long.class); - pythonToJavaTypeTemp.put("int16", short.class); - pythonToJavaTypeTemp.put("uint16", char.class); - pythonToJavaTypeTemp.put("uint8", byte.class); - pythonToJavaTypeTemp.put("int8", byte.class); - pythonToJavaTypeTemp.put("bool", Boolean.class); - pythonToJavaTypeTemp.put("bool_", Boolean.class); - pythonToJavaTypeTemp.put("object", Object.class); - pythonToJavaTypeTemp.put("unicode_type", String.class); - pythonToJavaType = Collections.unmodifiableMap(pythonToJavaTypeTemp); - - exec("import sys"); - pythonVersion = eval("sys.version.split()[0]").toString(); - if (pythonVersion.startsWith("2.")) { - // throw new RuntimeException("Numba support is only enabled for python 3 - current version is " + pythonVersion); - } - } - - public static Class toJavaType(String pythonType) { - if (pythonType.endsWith("[:]")) { - return Array.newInstance(pythonToJavaType.get(pythonType.substring(0, pythonType.length() - 3)), 0).getClass(); - } - if (pythonType.startsWith(NUMPY_ARRAY_PREFIX) && (pythonType.endsWith(NUMPY_ARRAY_SUFFIX_A) || pythonType.endsWith(NUMPY_ARRAY_SUFFIX_C))){ - return Array.newInstance(pythonToJavaType.get(pythonType.substring(NUMPY_ARRAY_PREFIX.length(), pythonType.length() - NUMPY_ARRAY_SUFFIX_A.length())), 0).getClass(); - } - return pythonToJavaType.get(pythonType); - } - - static LinkedHashMap findUsedColumns( - PyDictWrapper vectorized, - Map tableDefinition - ) { - final PyObject cols = vectorized.get("columns"); - final LinkedHashMap columns = new LinkedHashMap<>(); - for (PyObject used_col : cols.asList()) { - String colName = used_col.getStringValue(); - final ColumnDefinition col = tableDefinition.get(colName); - final String pyType = toPyType(col.getDataType(), col.getComponentType(), true); - columns.put(colName, pyType); - } - return columns; - } - - static Map getExpandedColumnNames(TableDefinition tableDefinition) { - Map columnNameMap = new LinkedHashMap<>(tableDefinition.getColumnNameMap()); - columnNameMap.put("i", ColumnDefinition.ofInt("i")); - columnNameMap.put("ii", ColumnDefinition.ofLong("ii")); - columnNameMap.put("k", ColumnDefinition.ofLong("k")); - return columnNameMap; - } - - static Map getExpandedColumnNames(Map tableDefinition) { - Map columnNameMap = new LinkedHashMap<>(tableDefinition); - columnNameMap.put("i", ColumnDefinition.ofInt("i")); - columnNameMap.put("ii", ColumnDefinition.ofLong("ii")); - columnNameMap.put("k", ColumnDefinition.ofLong("k")); - return columnNameMap; - } - - -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/NumbaFormulaColumn.java b/DB/src/main/java/io/deephaven/db/v2/select/NumbaFormulaColumn.java deleted file mode 100644 index f4c63318b19..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/NumbaFormulaColumn.java +++ /dev/null @@ -1,195 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.tables.ColumnDefinition; -import io.deephaven.db.tables.dbarrays.DbArrayBase; -import io.deephaven.db.tables.select.Param; -import io.deephaven.db.v2.select.formula.FormulaKernel; -import io.deephaven.db.v2.select.formula.FormulaKernelFactory; -import io.deephaven.db.v2.select.formula.FormulaSourceDescriptor; -import io.deephaven.db.v2.sources.chunk.*; -import io.deephaven.util.type.TypeUtils; -import org.jpy.PyDictWrapper; -import org.jpy.PyModule; -import org.jpy.PyObject; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import static io.deephaven.datastructures.util.CollectionUtil.ZERO_LENGTH_STRING_ARRAY; - -public class NumbaFormulaColumn extends AbstractFormulaColumn { - - private boolean initialized; - private LinkedHashMap usedCols; - private FormulaSourceDescriptor formulaSourceDescriptor; - private PyObject theFunction; - private Object returnValue; - private Map expandedColumnNameMap; - - /** - * Create a formula column for the given formula string. - *

- * The internal formula object is generated on-demand by calling out to the Java compiler. - * - * @param columnName the result column name - * @param formulaString the formula string to be parsed by the DBLanguageParser - */ - NumbaFormulaColumn(String columnName, String formulaString) { - super(columnName, formulaString, true); - } - - @Override - protected FormulaSourceDescriptor getSourceDescriptor() { - return formulaSourceDescriptor; - } - - @Override - protected FormulaKernelFactory getFormulaKernelFactory() { - - return new PythonKernelFactory(theFunction, usedCols.values().toArray(ZERO_LENGTH_STRING_ARRAY), - usedCols.keySet().stream().map(name -> expandedColumnNameMap.get(name).getDataType()).toArray(Class[]::new), returnedType, returnValue); - } - - @Override - public List initDef(Map columnNameMap) { - if (!initialized) { - try { - initialized = true; - expandedColumnNameMap = NumbaCompileTools.getExpandedColumnNames(columnNameMap); - PyDictWrapper vectorized = NumbaCompileTools.compileFormula(formulaString, expandedColumnNameMap); - usedCols = NumbaCompileTools.findUsedColumns(vectorized, expandedColumnNameMap); - - if (vectorized.containsKey("fun")) { - theFunction = vectorized.get("fun"); - } else { - theFunction = null; - returnValue = vectorized.get("result").getObjectValue(); - } - - Map finalColumnNameMap = expandedColumnNameMap; - Class[] usedColumnsTypes = usedCols.keySet().stream().map(name -> finalColumnNameMap.get(name).getDataType()).toArray(Class[]::new); - - returnedType = theFunction != null ? NumbaCompileTools.toJavaType(vectorized.get("return_type").toString()) : TypeUtils.getUnboxedType(returnValue.getClass()); - if (theFunction == null && returnedType == null) { - returnedType = returnValue.getClass(); - } - - formulaSourceDescriptor = new FormulaSourceDescriptor(returnedType, usedCols.keySet().toArray(ZERO_LENGTH_STRING_ARRAY), - ZERO_LENGTH_STRING_ARRAY, ZERO_LENGTH_STRING_ARRAY); - - - applyUsedVariables(expandedColumnNameMap, usedCols.keySet()); - - } catch (Exception e) { - throw new FormulaCompilationException("Formula compilation error for: " + formulaString, e); - } - } - return usedColumns; - } - - @Override - public SelectColumn copy() { - return new NumbaFormulaColumn(columnName, formulaString); - } - - private static class PythonKernelFactory implements FormulaKernelFactory { - - private final PyObject vectorized; - private final String[] columnsNpTypeName; - private final Class[] usedColumnsTypes; - private final Class returnType; - private final Object returnValue; - - private PythonKernelFactory(PyObject vectorized, String columnsNpTypeName[], Class[] usedColumnsTypes, Class returnType, Object returnValue) { - this.vectorized = vectorized; - this.columnsNpTypeName = columnsNpTypeName; - this.usedColumnsTypes = usedColumnsTypes; - this.returnType = returnType; - this.returnValue = returnValue; - } - - @Override - public FormulaKernel createInstance(DbArrayBase[] arrays, Param[] params) { - return new PythonKernel(vectorized, columnsNpTypeName, usedColumnsTypes, returnType, returnValue); - } - - } - - - private static class PythonKernel implements FormulaKernel { - private final PyObject vectorized; - private final Class[] usedColumnsTypes; - private final Class returnType; - private final PyModule np; - private final PyModule jpy; - private Object result; - private final NumbaCompileTools.NumbaEvaluator evaluator; - - public PythonKernel( - PyObject vectorized, - String[] columnsNpTypeName, Class[] usedColumnsTypes, Class returnType, Object constResult) { - this.vectorized = vectorized; - this.usedColumnsTypes = usedColumnsTypes; - this.returnType = returnType; - this.result = constResult; - this.np = PyModule.importModule("numpy"); - this.jpy = PyModule.importModule("jpy"); - evaluator = new NumbaCompileTools.NumbaEvaluator(vectorized, usedColumnsTypes, columnsNpTypeName); - } - - @Override - public Formula.FillContext makeFillContext(int maxChunkSize) { - return new Context(maxChunkSize, usedColumnsTypes, returnType); - } - - @Override - public void applyFormulaChunk(Formula.FillContext contextArg, WritableChunk destination, Chunk[] inputChunks) { - Context context = (Context) contextArg; - final int size = destination.size(); - if (vectorized == null) { - destination.fillWithBoxedValue(0, size, result); - return; - } - - try (final PyObject result = evaluator.numbaEvaluate(context.evaluatorContext, inputChunks, size)) { - if (returnType == Boolean.class) { - final Boolean[] objVal = result.getObjectArrayValue(Boolean.class); - WritableObjectChunk dest = destination.asWritableObjectChunk(); - for (int i = 0; i < destination.size(); i++) { - dest.set(i, objVal[i]); - } - } else { - // todo: this won't work, I think we need to make sure it's a *primitive* array. - //final Object[] objVal = result.getObjectArrayValue(returnType); - - final Object objVal; - try (final PyObject pyObj = jpy.call("array", returnType.getName(), result)) { - objVal = pyObj.getObjectValue(); - } - destination.copyFromArray(objVal, 0, 0, destination.size()); - } - } - } - - class Context implements Formula.FillContext { - - public final int chunkSize; - private final WritableChunk resultChunk; - private final NumbaCompileTools.NumbaEvaluator.Context evaluatorContext; - - - public Context(int maxChunkSize, Class[] usedColumnsTypes, Class returnType) { - resultChunk = ChunkType.fromElementType(returnType).makeWritableChunk(maxChunkSize); - this.chunkSize = maxChunkSize; - this.evaluatorContext = evaluator.getContext(); - } - - @Override - public void close() { - resultChunk.close(); - } - } - - } -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArgumentsBuilder.java b/DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArgumentsBuilder.java deleted file mode 100644 index 300553d6cbb..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArgumentsBuilder.java +++ /dev/null @@ -1,49 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.v2.sources.chunk.Chunk; -import org.jpy.PyModule; - -import java.lang.reflect.Array; - -public class PrimitiveArgumentsBuilder implements ArgumentsBuilder { - - private final String npTypeName; - - class Context implements ArgumentsBuilder.Context { - private final PyModule np; - private final Object npType; - Object tentativeDestination; - int currentSize = 0; - Context() { - this.np = PyModule.importModule("numpy"); - this.npType = np.getAttribute(npTypeName); - } - } - - private final Class javaType; - - public PrimitiveArgumentsBuilder(String npTypeName, Class javaType) { - this.javaType = javaType; - this.npTypeName = npTypeName; - } - - - @Override - public int getArgumentsCount() { - return 1; - } - - @Override - public Object[] packArguments(Chunk incomingData, Context context, int size) { - Object destArray = context.tentativeDestination != null && context.currentSize == size ? - context.tentativeDestination : Array.newInstance(javaType, size); - context.tentativeDestination = destArray; - incomingData.copyToArray(0, destArray, 0, size); - return new Object[]{context.np.call("asarray", destArray, context.npType)}; - } - - @Override - public Context getContext() { - return new Context(); - } -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArrayArgumentsBuilder.java b/DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArrayArgumentsBuilder.java deleted file mode 100644 index 78880de248b..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/PrimitiveArrayArgumentsBuilder.java +++ /dev/null @@ -1,67 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.v2.sources.chunk.*; -import org.jpy.PyModule; -import org.jpy.PyObject; - -import java.lang.reflect.Array; - -public class PrimitiveArrayArgumentsBuilder implements ArgumentsBuilder { - - private final Class javaType; - private final String npElementTypeName; - - class Context implements ArgumentsBuilder.Context { - - private final PyObject npInt32; - private final PyModule np; - private final Object npType; - - int[] tentativeOffsetsDestination; - Context(){ - this.np = PyModule.importModule("numpy"); - this.npType = np.getAttribute(npElementTypeName); - this.npInt32 = np.getAttribute("int32"); - } - } - - public PrimitiveArrayArgumentsBuilder(String npTypeName) { - npElementTypeName = npTypeName.substring(0, npTypeName.length() - 3); - this.javaType = NumbaCompileTools.toJavaType(npElementTypeName); - } - - @Override - public int getArgumentsCount() { - return 2; - } - - @Override - public Object[] packArguments(Chunk incomingData, Context context, int size) { - int[] offsetDest = context.tentativeOffsetsDestination != null && context.tentativeOffsetsDestination.length == size ? - context.tentativeOffsetsDestination : new int[size]; - context.tentativeOffsetsDestination = offsetDest; - int totalSize = 0; - ObjectChunk typedChunk = (ObjectChunk) incomingData; - for (int i = 0; i < size; i++) { - totalSize += Array.getLength(typedChunk.get(i)); - - } - Object destArray = Array.newInstance(javaType, totalSize); - WritableChunk chunkDest = ChunkType.fromElementType(javaType).writableChunkWrap(destArray, 0, totalSize); - int offset = 0; - for (int i = 0; i < size; i++) { - Object srcArray = typedChunk.get(i); - int len = Array.getLength(srcArray); - chunkDest.copyFromArray(srcArray,0,offset,len); - offset += len; - offsetDest[i] = offset; - } - - return new Object[]{context.np.call("asarray", destArray, context.npType),context.np.call("asarray", offsetDest, context.npInt32)}; - } - - @Override - public Context getContext() { - return new Context(); - } -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/PythonVectorFilter.java b/DB/src/main/java/io/deephaven/db/v2/select/PythonVectorFilter.java deleted file mode 100644 index 173d9584db0..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/PythonVectorFilter.java +++ /dev/null @@ -1,164 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.tables.ColumnDefinition; -import io.deephaven.db.tables.Table; -import io.deephaven.db.tables.TableDefinition; -import io.deephaven.db.tables.lang.DBLanguageParser; -import io.deephaven.db.tables.utils.DBTimeUtils.Result; -import io.deephaven.db.v2.select.ConditionFilter.ChunkFilter; -import io.deephaven.db.v2.select.ConditionFilter.FilterKernel; -import io.deephaven.db.v2.sources.chunk.Attributes; -import io.deephaven.db.v2.sources.chunk.Chunk; -import io.deephaven.db.v2.sources.chunk.LongChunk; -import io.deephaven.db.v2.utils.Index; -import io.deephaven.jpy.JpyModule; -import org.jetbrains.annotations.NotNull; -import org.jpy.*; - -import java.util.LinkedHashMap; -import java.util.Map; - -import static io.deephaven.datastructures.util.CollectionUtil.ZERO_LENGTH_STRING_ARRAY; - -/** - * A {@link AbstractConditionFilter} which transfers primitive data to python in bulk, - * which is sent as numba arrays into a bulk `@vectorize`d function, - * where the entire loop is processed in native C. - *

- * For very complex expressions, we may want to consider using @guvectorize and cuda mode, - * which would be able to process large chunks of data in parallel, in C. - */ -public class PythonVectorFilter extends AbstractConditionFilter { - - private PyDictWrapper vectorized; - private LinkedHashMap usedCols; - - protected PythonVectorFilter(@NotNull String formula) { - super(formula, false); - } - - private PythonVectorFilter(@NotNull String formula, Map renames) { - super(formula, renames, false); - } - - @Override - public synchronized void init(TableDefinition tableDefinition) { - - if (initialized) { - return; - } - initialized = true; - vectorized = NumbaCompileTools.compileFormula(formula, NumbaCompileTools.getExpandedColumnNames(tableDefinition)); - if (!vectorized.get("return_type").toString().equals("bool")) { - throw new FormulaCompilationException("Expression " + formula + " return type is not bool but " + vectorized.get("return_type")); - } - usedCols = NumbaCompileTools.findUsedColumns(vectorized, NumbaCompileTools.getExpandedColumnNames(tableDefinition)); - } - - @Override - public boolean isRefreshing() { - return false; - } - - @Override - public boolean canMemoize() { - return false; - } - - @Override - protected void generateFilterCode( - TableDefinition tableDefinition, Result timeConversionResult, DBLanguageParser.Result result - ) { - throw new UnsupportedOperationException("Python filter does not generate java code"); - } - - @Override - protected Filter getFilter(Table table, Index fullSet) { - final Map columnDefMap = NumbaCompileTools.getExpandedColumnNames(table.getDefinition()); - final Class[] usedColumnsTypes = usedCols.keySet().stream().map(name->columnDefMap.get(name).getDataType()).toArray(Class[]::new); - return new ChunkFilter(new PythonVectorKernel(vectorized.get("fun"), usedCols, usedColumnsTypes), usedCols.keySet().toArray(ZERO_LENGTH_STRING_ARRAY), ConditionFilter.CHUNK_SIZE); - } - - @Override - public AbstractConditionFilter copy() { - // Hm. should send the already-compiled function? ...probably - return new PythonVectorFilter(formula, outerToInnerNames); - } - - @Override - public AbstractConditionFilter renameFilter(Map renames) { - // This one probably shouldn't send the already-compiled function... - return new PythonVectorFilter(formula, renames); - } - - - /** - * This class is responsible for actually executing the vectorized function at runtime. - *

- * It will perform bulk transfer-to-python operations on the data, - * then execute the vectorized function on the results, - * and return an index of all matching items. - */ - static class PythonVectorKernel implements FilterKernel { - - private final NumbaCompileTools.NumbaEvaluator numbaEvaluator; - private final JpyModule jpy; - - PythonVectorKernel( - PyObject vectorized, - LinkedHashMap usedColumns, Class[] usedColumnsTypes) { - numbaEvaluator = new NumbaCompileTools.NumbaEvaluator(vectorized,usedColumnsTypes,usedColumns.values().toArray(new String[0])); - jpy = JpyModule.create(); - } - - class Context extends FilterKernel.Context { - - private final NumbaCompileTools.NumbaEvaluator.Context numbaEvaluatorContext; - - Context(int maxChunkSize) { - super(maxChunkSize); - numbaEvaluatorContext = numbaEvaluator.getContext(); - } - - @Override - public void close() { - resultChunk.close(); - } - } - - @Override - public Context getContext(int maxChunkSize) { - return new Context(maxChunkSize); - } - - @Override - public LongChunk filter(Context context, LongChunk indices, Chunk... inputChunks) { - final int size = indices.size(); - - // TODO: IDS-6241 - // this results in: Value: cannot convert a Python 'numpy.bool_' to a Java 'java.lang.Object' - /* - final Boolean[] matches; - try (final PyObject result = numbaEvaluator.numbaEvaluate(context.numbaEvaluatorContext,inputChunks,size)) { - matches = result.getObjectArrayValue(Boolean.class); - } - */ - - // Regardless, this is *hacky*, even without resorting to jpy.to_boolean_array. - // numbaEvaluator.numbaEvaluate *should* be responsible for returning proper types - final boolean[] matches; - try (final PyObject result = numbaEvaluator.numbaEvaluate(context.numbaEvaluatorContext,inputChunks,size)) { - matches = jpy.to_boolean_array(result); - } - - context.resultChunk.setSize(0); - for (int i = 0; i < matches.length; i++) { - if (matches[i]) { - context.resultChunk.add(indices.get(i)); - } - } - return context.resultChunk; - } - - } -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/StringArgumentsBuilder.java b/DB/src/main/java/io/deephaven/db/v2/select/StringArgumentsBuilder.java deleted file mode 100644 index 024577308d2..00000000000 --- a/DB/src/main/java/io/deephaven/db/v2/select/StringArgumentsBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -package io.deephaven.db.v2.select; - -import io.deephaven.db.v2.sources.chunk.Attributes; -import io.deephaven.db.v2.sources.chunk.Chunk; -import io.deephaven.db.v2.sources.chunk.ObjectChunk; -import org.jpy.PyModule; -import org.jpy.PyObject; - -public class StringArgumentsBuilder implements ArgumentsBuilder { - - - class Context implements ArgumentsBuilder.Context { - - private final PyObject npInt32; - private final PyModule np; - - int[] tentativeOffsetsDestination; - StringBuilder stringBuilder = new StringBuilder(); - Context(){ - this.np = PyModule.importModule("numpy"); - this.npInt32 = np.getAttribute("int32"); - } - } - - public StringArgumentsBuilder() { - } - - - @Override - public int getArgumentsCount() { - return 2; - } - - @Override - public Object[] packArguments(Chunk incomingData, Context context, int size) { - int[] offsetDest = context.tentativeOffsetsDestination != null && context.tentativeOffsetsDestination.length == size ? - context.tentativeOffsetsDestination : new int[size]; - context.tentativeOffsetsDestination = offsetDest; - context.stringBuilder.setLength(0); - ObjectChunk typedChunk = (ObjectChunk) incomingData; - for (int i = 0; i < size;i++) { - context.stringBuilder.append(typedChunk.get(i)); - offsetDest[i] = context.stringBuilder.length(); - } - - return new Object[]{context.stringBuilder.toString(),context.np.call("asarray", offsetDest, context.npInt32)}; - } - - @Override - public Context getContext() { - return new Context(); - } -} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsChunked.java b/DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsChunked.java new file mode 100644 index 00000000000..5ed0fb0f8b4 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsChunked.java @@ -0,0 +1,124 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.sources.chunk.Attributes.Any; +import io.deephaven.db.v2.sources.chunk.*; +import io.deephaven.util.PrimitiveArrayType; + +import java.util.Objects; + +class ArgumentsChunked { + static ArgumentsChunked buildArguments(Chunk[] __sources) { + final Class[] paramTypes = new Class[__sources.length]; + final Object[] params = new Object[__sources.length]; + for (int i = 0; i < __sources.length; i++) { + final ChunkToArray cta = __sources[i].walk(new ChunkToArray<>()); + paramTypes[i] = Objects.requireNonNull(cta.getArrayType()); + params[i] = Objects.requireNonNull(cta.getArray()); + } + return new ArgumentsChunked(paramTypes, params); + } + + private final Class[] paramTypes; + private final Object[] params; + + private ArgumentsChunked(Class[] paramTypes, Object[] params) { + this.paramTypes = paramTypes; + this.params = params; + } + + Class[] getParamTypes() { + return paramTypes; + } + + Object[] getParams() { + return params; + } + + private static class ChunkToArray implements Chunk.Visitor { + + private Class arrayType; + private Object array; + + Class getArrayType() { + return Objects.requireNonNull(arrayType); + } + + Object getArray() { + return Objects.requireNonNull(array); + } + + @Override + public void visit(ByteChunk chunk) { + arrayType = PrimitiveArrayType.bytes().getArrayType(); + final byte[] out = PrimitiveArrayType.bytes().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(BooleanChunk chunk) { + arrayType = PrimitiveArrayType.booleans().getArrayType(); + final boolean[] out = PrimitiveArrayType.booleans().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(CharChunk chunk) { + arrayType = PrimitiveArrayType.chars().getArrayType(); + final char[] out = PrimitiveArrayType.chars().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(ShortChunk chunk) { + arrayType = PrimitiveArrayType.shorts().getArrayType(); + final short[] out = PrimitiveArrayType.shorts().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(IntChunk chunk) { + arrayType = PrimitiveArrayType.ints().getArrayType(); + final int[] out = PrimitiveArrayType.ints().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(LongChunk chunk) { + arrayType = PrimitiveArrayType.longs().getArrayType(); + final long[] out = PrimitiveArrayType.longs().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(FloatChunk chunk) { + arrayType = PrimitiveArrayType.floats().getArrayType(); + final float[] out = PrimitiveArrayType.floats().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(DoubleChunk chunk) { + arrayType = PrimitiveArrayType.doubles().getArrayType(); + final double[] out = PrimitiveArrayType.doubles().newInstance(chunk.size()); + chunk.copyToTypedArray(0, out, 0, out.length); + array = out; + } + + @Override + public void visit(ObjectChunk chunk) { + // this is LESS THAN IDEAL - it would be much better if ObjectChunk would be able to return + // the array type + arrayType = Object[].class; + final Object[] out = new Object[chunk.size()]; + chunk.copyToArray(0, out, 0, out.length); + array = out; + } + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsSingular.java b/DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsSingular.java new file mode 100644 index 00000000000..d4c959e149e --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/ArgumentsSingular.java @@ -0,0 +1,141 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.sources.chunk.Attributes.Any; +import io.deephaven.db.v2.sources.chunk.*; +import io.deephaven.db.v2.sources.chunk.Chunk.Visitor; +import io.deephaven.util.PrimitiveArrayType; + +import java.util.Objects; +import java.util.stream.Stream; + +class ArgumentsSingular { + + static Class[] buildParamTypes(Chunk[] __sources) { + return Stream.of(__sources) + .map(c -> c.walk(new ChunkToSingularType<>())) + .map(ChunkToSingularType::getOut) + .toArray(Class[]::new); + } + + static Object[] buildArguments(Chunk[] __sources, int index) { + return Stream.of(__sources) + .map(c -> c.walk(new ChunkIndexToObject<>(index))) + .map(ChunkIndexToObject::getOut) + .toArray(); + } + + private static class ChunkIndexToObject implements Visitor { + + private final int index; + + private Object out; + + ChunkIndexToObject(int index) { + this.index = index; + } + + Object getOut() { + return out; + } + + @Override + public void visit(ByteChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(BooleanChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(CharChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(ShortChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(IntChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(LongChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(FloatChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(DoubleChunk chunk) { + out = chunk.get(index); + } + + @Override + public void visit(ObjectChunk chunk) { + out = chunk.get(index); + } + } + + private static class ChunkToSingularType implements Visitor { + private Class out; + + Class getOut() { + return Objects.requireNonNull(out); + } + + @Override + public void visit(ByteChunk chunk) { + out = PrimitiveArrayType.bytes().getPrimitiveType(); + } + + @Override + public void visit(BooleanChunk chunk) { + out = PrimitiveArrayType.booleans().getPrimitiveType(); + } + + @Override + public void visit(CharChunk chunk) { + out = PrimitiveArrayType.chars().getPrimitiveType(); + } + + @Override + public void visit(ShortChunk chunk) { + out = PrimitiveArrayType.shorts().getPrimitiveType(); + } + + @Override + public void visit(IntChunk chunk) { + out = PrimitiveArrayType.ints().getPrimitiveType(); + } + + @Override + public void visit(LongChunk chunk) { + out = PrimitiveArrayType.longs().getPrimitiveType(); + } + + @Override + public void visit(FloatChunk chunk) { + out = PrimitiveArrayType.floats().getPrimitiveType(); + } + + @Override + public void visit(DoubleChunk chunk) { + out = PrimitiveArrayType.doubles().getPrimitiveType(); + } + + @Override + public void visit(ObjectChunk chunk) { + // this is LESS THAN IDEAL - it would be much better if ObjectChunk would be able to return + // the item type + out = Object.class; + } + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/ConditionFilterPython.java b/DB/src/main/java/io/deephaven/db/v2/select/python/ConditionFilterPython.java new file mode 100644 index 00000000000..5553b9c71ca --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/ConditionFilterPython.java @@ -0,0 +1,74 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.tables.Table; +import io.deephaven.db.tables.TableDefinition; +import io.deephaven.db.tables.lang.DBLanguageParser; +import io.deephaven.db.tables.utils.DBTimeUtils.Result; +import io.deephaven.db.v2.select.AbstractConditionFilter; +import io.deephaven.db.v2.select.ConditionFilter; +import io.deephaven.db.v2.select.ConditionFilter.ChunkFilter; +import io.deephaven.db.v2.utils.Index; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +/** + * A condition filter for python native code. + */ +public class ConditionFilterPython extends AbstractConditionFilter { + + @SuppressWarnings("unused") // called from python + public static ConditionFilterPython create(DeephavenCompatibleFunction dcf) { + return new ConditionFilterPython(dcf); + } + + private final DeephavenCompatibleFunction dcf; + + private ConditionFilterPython(DeephavenCompatibleFunction dcf) { + this(Collections.emptyMap(), dcf); + } + + private ConditionFilterPython(Map renames, DeephavenCompatibleFunction dcf) { + super("", renames, false); + this.dcf = Objects.requireNonNull(dcf); + } + + @Override + public void init(TableDefinition tableDefinition) { + // no-op + } + + @Override + protected Filter getFilter(Table table, Index fullSet) { + return new ChunkFilter( + dcf.toFilterKernel(), + dcf.getColumnNames().toArray(new String[0]), + ConditionFilter.CHUNK_SIZE); + } + + @Override + public AbstractConditionFilter copy() { + return new ConditionFilterPython(dcf); + } + + @Override + public AbstractConditionFilter renameFilter(Map renames) { + return new ConditionFilterPython(renames, dcf); + } + + @Override + public boolean isRefreshing() { + return false; + } + + @Override + public boolean canMemoize() { + return false; + } + + @Override + protected void generateFilterCode(TableDefinition tableDefinition, Result timeConversionResult, DBLanguageParser.Result result) { + throw new UnsupportedOperationException("ConditionFilterPython does not generate java code"); + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/DeephavenCompatibleFunction.java b/DB/src/main/java/io/deephaven/db/v2/select/python/DeephavenCompatibleFunction.java new file mode 100644 index 00000000000..ee8a436a4a8 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/DeephavenCompatibleFunction.java @@ -0,0 +1,79 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.select.ConditionFilter.FilterKernel; +import io.deephaven.db.v2.select.ConditionFilter.FilterKernel.Context; +import io.deephaven.db.v2.select.formula.FormulaKernel; +import org.jpy.PyObject; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * A Deephaven-compatible functions holds a native python function with associated typing + * information, used to help implement {@link io.deephaven.db.v2.select.python.ConditionFilterPython} and {@link FormulaColumnPython}. + */ +public class DeephavenCompatibleFunction { + + @SuppressWarnings("unused") // called from python + public static DeephavenCompatibleFunction create( + PyObject function, + + // todo: python can't convert from java type to Class (ie, java_func_on_type(jpy.get_type('...'))) + // but it *will* match on object, and unwrap the actual java type... + Object returnedType, + + // todo: python can't convert from list of strings to List + // but it can convert from list of strings to String[]... + String[] columnNames, + boolean isVectorized) { + return new DeephavenCompatibleFunction(function, (Class)returnedType, Arrays.asList(columnNames), isVectorized); + } + + private final PyObject function; + private final Class returnedType; // the un-vectorized type (if this function is vectorized) + private final List columnNames; + private final boolean isVectorized; + + private DeephavenCompatibleFunction( + PyObject function, + Class returnedType, + List columnNames, + boolean isVectorized) { + this.function = Objects.requireNonNull(function, "function"); + this.returnedType = Objects.requireNonNull(returnedType, "returnedType"); + this.columnNames = Objects.requireNonNull(columnNames, "columnNames"); + this.isVectorized = isVectorized; + } + + public FormulaKernel toFormulaKernel() { + return isVectorized ? + new FormulaKernelPythonChunkedFunction(function) : + new io.deephaven.db.v2.select.python.FormulaKernelPythonSingularFunction(function); + } + + public FilterKernel toFilterKernel() { + if (returnedType != boolean.class) { + throw new IllegalStateException("FilterKernel functions must be annotated with a boolean return type"); + } + return isVectorized ? + new FilterKernelPythonChunkedFunction(function) : + new FilterKernelPythonSingularFunction(function); + } + + public PyObject getFunction() { + return function; + } + + public Class getReturnedType() { + return returnedType; + } + + public List getColumnNames() { + return columnNames; + } + + public boolean isVectorized() { + return isVectorized; + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/FillContextPython.java b/DB/src/main/java/io/deephaven/db/v2/select/python/FillContextPython.java new file mode 100644 index 00000000000..4c523f7c948 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/FillContextPython.java @@ -0,0 +1,12 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.select.Formula.FillContext; + +enum FillContextPython implements FillContext { + EMPTY; + + @Override + public void close() { + // ignore + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonChunkedFunction.java b/DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonChunkedFunction.java new file mode 100644 index 00000000000..30070eff751 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonChunkedFunction.java @@ -0,0 +1,53 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.select.ConditionFilter.FilterKernel; +import io.deephaven.db.v2.sources.chunk.Attributes.OrderedKeyIndices; +import io.deephaven.db.v2.sources.chunk.Chunk; +import io.deephaven.db.v2.sources.chunk.LongChunk; +import org.jpy.PyObject; + +import java.util.Objects; + +/** + * A python filter kernel which is implemented by passing the chunks as arrays into the python + * function. + * + * @see io.deephaven.db.v2.select.python.FilterKernelPythonSingularFunction + */ +class FilterKernelPythonChunkedFunction implements FilterKernel { + + private static final String CALL_METHOD = "__call__"; + + // this is a python function whose arguments can accept arrays + private final PyObject function; + + FilterKernelPythonChunkedFunction(PyObject function) { + this.function = Objects.requireNonNull(function, "function"); + } + + @Override + public Context getContext(int maxChunkSize) { + return new Context(maxChunkSize); + } + + @Override + public LongChunk filter( + Context context, + LongChunk indices, + Chunk... inputChunks) { + final int size = indices.size(); + final io.deephaven.db.v2.select.python.ArgumentsChunked arguments = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(inputChunks); + final boolean[] results = function + .call(boolean[].class, CALL_METHOD, arguments.getParamTypes(), arguments.getParams()); + if (size != results.length) { + throw new IllegalStateException("FilterKernelPythonChunkedFunction returned results are not the proper size"); + } + context.resultChunk.setSize(0); + for (int i = 0; i < size; ++i) { + if (results[i]) { + context.resultChunk.add(indices.get(i)); + } + } + return context.resultChunk; + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonSingularFunction.java b/DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonSingularFunction.java new file mode 100644 index 00000000000..41a1c14affa --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/FilterKernelPythonSingularFunction.java @@ -0,0 +1,48 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.select.ConditionFilter.FilterKernel; +import io.deephaven.db.v2.sources.chunk.Attributes.OrderedKeyIndices; +import io.deephaven.db.v2.sources.chunk.Chunk; +import io.deephaven.db.v2.sources.chunk.LongChunk; +import org.jpy.PyObject; + +import java.util.Objects; + +/** + * A python filter kernel which is implemented by iterating over the input chunks and calling the + * python function N times. + * + * @see FilterKernelPythonChunkedFunction + */ +class FilterKernelPythonSingularFunction implements FilterKernel { + + private static final String CALL_METHOD = "__call__"; + + private final PyObject function; + + FilterKernelPythonSingularFunction(PyObject function) { + this.function = Objects.requireNonNull(function, "function"); + } + + @Override + public Context getContext(int maxChunkSize) { + return new Context(maxChunkSize); + } + + @Override + public LongChunk filter( + Context context, + LongChunk indices, + Chunk... inputChunks) { + final int size = indices.size(); + final Class[] paramTypes = ArgumentsSingular.buildParamTypes(inputChunks); + context.resultChunk.setSize(0); + for (int i = 0; i < size; ++i) { + final Object[] params = ArgumentsSingular.buildArguments(inputChunks, i); + if (function.call(boolean.class, CALL_METHOD, paramTypes, params)) { + context.resultChunk.add(indices.get(i)); + } + } + return context.resultChunk; + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaColumnPython.java b/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaColumnPython.java new file mode 100644 index 00000000000..7b1bb3d13b6 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaColumnPython.java @@ -0,0 +1,86 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.tables.ColumnDefinition; +import io.deephaven.db.tables.dbarrays.DbArrayBase; +import io.deephaven.db.tables.select.Param; +import io.deephaven.db.v2.select.AbstractFormulaColumn; +import io.deephaven.db.v2.select.SelectColumn; +import io.deephaven.db.v2.select.formula.FormulaKernel; +import io.deephaven.db.v2.select.formula.FormulaKernelFactory; +import io.deephaven.db.v2.select.formula.FormulaSourceDescriptor; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import io.deephaven.datastructures.util.CollectionUtil; + +import static io.deephaven.datastructures.util.CollectionUtil.ZERO_LENGTH_STRING_ARRAY; + +/** + * A formula column for python native code. + */ +public class FormulaColumnPython extends AbstractFormulaColumn implements FormulaKernelFactory { + + @SuppressWarnings("unused") // called from python + public static FormulaColumnPython create(String columnName, io.deephaven.db.v2.select.python.DeephavenCompatibleFunction dcf) { + return new FormulaColumnPython(columnName, dcf); + } + + private final io.deephaven.db.v2.select.python.DeephavenCompatibleFunction dcf; + + private boolean initialized; + + private FormulaColumnPython(String columnName, io.deephaven.db.v2.select.python.DeephavenCompatibleFunction dcf) { + super(columnName, "", true); + this.dcf = Objects.requireNonNull(dcf); + } + + private void initFromDef(Map columnNameMap) { + if (initialized) { + throw new IllegalStateException("Already initialized"); + } + returnedType = dcf.getReturnedType(); + this.initialized = true; + } + + @Override + protected final FormulaKernelFactory getFormulaKernelFactory() { + return this; + } + + @Override + public final List initDef(Map columnNameMap) { + if (!initialized) { + initFromDef(columnNameMap); + applyUsedVariables(columnNameMap, new LinkedHashSet<>(dcf.getColumnNames())); + } + return usedColumns; + } + + @Override + protected final FormulaSourceDescriptor getSourceDescriptor() { + if (!initialized) { + throw new IllegalStateException("Must be initialized first"); + } + return new FormulaSourceDescriptor( + returnedType, + dcf.getColumnNames().toArray(new String[0]), + ZERO_LENGTH_STRING_ARRAY, + ZERO_LENGTH_STRING_ARRAY); + } + + @Override + public final SelectColumn copy() { + return new FormulaColumnPython(columnName, dcf); + } + + @Override + public final FormulaKernel createInstance(DbArrayBase[] arrays, Param[] params) { + if (!initialized) { + throw new IllegalStateException("Must be initialized first"); + } + return dcf.toFormulaKernel(); + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonChunkedFunction.java b/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonChunkedFunction.java new file mode 100644 index 00000000000..7dc9dad5766 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonChunkedFunction.java @@ -0,0 +1,137 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.select.Formula.FillContext; +import io.deephaven.db.v2.select.FormulaKernelTypedBase; +import io.deephaven.db.v2.select.formula.FormulaKernel; +import io.deephaven.db.v2.sources.chunk.*; +import io.deephaven.db.v2.sources.chunk.Attributes.Values; +import org.jpy.PyObject; + +import java.util.Objects; + +/** + * A python formula kernel which is implemented by passing the chunks as arrays into the python + * function. + * + * @see io.deephaven.db.v2.select.python.FormulaKernelPythonSingularFunction + */ +class FormulaKernelPythonChunkedFunction extends FormulaKernelTypedBase implements FormulaKernel { + + private static final String CALL_METHOD = "__call__"; + + // this is a python function whose arguments can accept arrays + private final PyObject function; + + FormulaKernelPythonChunkedFunction(PyObject function) { + this.function = Objects.requireNonNull(function, "function"); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableByteChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final byte[] output = function + .call(byte[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableBooleanChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final boolean[] output = function + .call(boolean[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableCharChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final char[] output = function + .call(char[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableShortChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final short[] output = function + .call(short[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableIntChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final int[] output = function + .call(int[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableLongChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final long[] output = function + .call(long[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableFloatChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final float[] output = function + .call(float[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableDoubleChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + final double[] output = function + .call(double[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + __destination.copyFromTypedArray(output, 0, 0, output.length); + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableObjectChunk __destination, + Chunk[] __sources) { + final io.deephaven.db.v2.select.python.ArgumentsChunked args = io.deephaven.db.v2.select.python.ArgumentsChunked.buildArguments(__sources); + + // this is LESS THAN IDEAL - it would be much better if ObjectChunk would be able to return + // the array type + final Object[] output = function + .call(Object[].class, CALL_METHOD, args.getParamTypes(), args.getParams()); + + //noinspection unchecked + __destination.copyFromTypedArray((T[]) output, 0, 0, output.length); + } + + @Override + public FillContext makeFillContext(int __chunkCapacity) { + return FillContextPython.EMPTY; + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonSingularFunction.java b/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonSingularFunction.java new file mode 100644 index 00000000000..53fdc5005a7 --- /dev/null +++ b/DB/src/main/java/io/deephaven/db/v2/select/python/FormulaKernelPythonSingularFunction.java @@ -0,0 +1,164 @@ +package io.deephaven.db.v2.select.python; + +import io.deephaven.db.v2.select.Formula.FillContext; +import io.deephaven.db.v2.select.FormulaKernelTypedBase; +import io.deephaven.db.v2.select.formula.FormulaKernel; +import io.deephaven.db.v2.sources.chunk.Attributes.Values; +import io.deephaven.db.v2.sources.chunk.*; +import org.jpy.PyObject; + +import java.util.Objects; + +/** + * A python formula kernel which is implemented by iterating over the input chunks and calling the + * python function N times. + * + * @see FormulaKernelPythonChunkedFunction + */ +class FormulaKernelPythonSingularFunction extends FormulaKernelTypedBase implements FormulaKernel { + + private static final String CALL_METHOD = "__call__"; + + private final PyObject function; + + FormulaKernelPythonSingularFunction(PyObject function) { + this.function = Objects.requireNonNull(function, "function"); + } + + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableByteChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final byte output = function + .call(byte.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableBooleanChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final boolean output = function + .call(boolean.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableCharChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final char output = function + .call(char.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableShortChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final short output = function + .call(short.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableIntChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final int output = function + .call(int.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableLongChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final long output = function + .call(long.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableFloatChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final float output = function + .call(float.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableDoubleChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + final double output = function + .call(double.class, CALL_METHOD, types, ArgumentsSingular.buildArguments(__sources, i)); + __destination.set(i, output); + } + } + + @Override + public void applyFormulaChunk( + FillContext __context, + WritableObjectChunk __destination, + Chunk[] __sources) { + final Class[] types = ArgumentsSingular.buildParamTypes(__sources); + final int L = __destination.size(); + for (int i = 0; i < L; i++) { + // this is LESS THAN IDEAL - it would be much better if ObjectChunk would be able to return + // the non-array type + final Object output = function + .call(Object.class, CALL_METHOD, types, ArgumentsSingular + .buildArguments(__sources, i)); + + //noinspection unchecked + __destination.set(i, (T)output); + } + } + + @Override + public FillContext makeFillContext(int __chunkCapacity) { + return FillContextPython.EMPTY; + } +} diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/BooleanChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/BooleanChunk.java index 3d179053c73..b0d326e16e6 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/BooleanChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/BooleanChunk.java @@ -114,6 +114,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ByteChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ByteChunk.java index 71d4537e1b6..9fc5f6e7977 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ByteChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ByteChunk.java @@ -117,6 +117,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder public final T applyDecoder(ObjectDecoder decoder) { return decoder.decode(data, offset, size); diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/CharChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/CharChunk.java index 21068739251..ce18c91702b 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/CharChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/CharChunk.java @@ -113,6 +113,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/Chunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/Chunk.java index 5b36bba37a5..6972514c79b 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/Chunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/Chunk.java @@ -26,6 +26,18 @@ public interface Chunk { */ int MAXIMUM_SIZE = Integer.MAX_VALUE; + interface Visitor { + void visit(ByteChunk chunk); + void visit(BooleanChunk chunk); + void visit(CharChunk chunk); + void visit(ShortChunk chunk); + void visit(IntChunk chunk); + void visit(LongChunk chunk); + void visit(FloatChunk chunk); + void visit(DoubleChunk chunk); + void visit(ObjectChunk chunk); + } + /** * Make a new Chunk that represents either exactly the same view on the underlying data as this Chunk, or a * subrange of that view. The view is defined as [0..size) (in the coordinate space of this Chunk). @@ -87,6 +99,13 @@ default void copyToBuffer(int srcOffset, @NotNull Buffer destBuffer, int destOff */ ChunkType getChunkType(); + default void checkChunkType(ChunkType expected) { + final ChunkType actual = getChunkType(); + if (actual != expected) { + throw new IllegalArgumentException(String.format("Expected chunk type '%s', but is '%s'.", expected, actual)); + } + } + /** * @return true iff this and array are aliases, that is they refer to the same underlying data */ @@ -97,6 +116,8 @@ default void copyToBuffer(int srcOffset, @NotNull Buffer destBuffer, int destOff */ boolean isAlias(Chunk chunk); + > V walk(V visitor); + default ByteChunk asByteChunk() { return (ByteChunk) this; } diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/DoubleChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/DoubleChunk.java index 77ff75010b9..d4828000634 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/DoubleChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/DoubleChunk.java @@ -116,6 +116,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/FloatChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/FloatChunk.java index feec8e4622e..61298e78696 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/FloatChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/FloatChunk.java @@ -116,6 +116,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/IntChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/IntChunk.java index 7fc1696c866..9b09183cc3b 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/IntChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/IntChunk.java @@ -116,6 +116,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/LongChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/LongChunk.java index 85adf2761fa..a7a845c489e 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/LongChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/LongChunk.java @@ -116,6 +116,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ObjectChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ObjectChunk.java index 510b6edb749..2a1bc8ac11b 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ObjectChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ObjectChunk.java @@ -116,6 +116,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ShortChunk.java b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ShortChunk.java index 5db09f5161a..5e25d4836da 100644 --- a/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ShortChunk.java +++ b/DB/src/main/java/io/deephaven/db/v2/sources/chunk/ShortChunk.java @@ -116,6 +116,12 @@ public final boolean isAlias(Chunk chunk) { return chunk.isAlias(data); } + @Override + public final > V walk(V visitor) { + visitor.visit(this); + return visitor; + } + // region ApplyDecoder // endregion ApplyDecoder diff --git a/DB/src/main/java/io/deephaven/python/ASTHelper.java b/DB/src/main/java/io/deephaven/python/ASTHelper.java deleted file mode 100644 index a5592b659e3..00000000000 --- a/DB/src/main/java/io/deephaven/python/ASTHelper.java +++ /dev/null @@ -1,55 +0,0 @@ -package io.deephaven.python; - -import java.util.List; -import java.util.stream.Collectors; -import org.jpy.PyObject; - -/** - * See ast_helper.py - */ -public interface ASTHelper extends AutoCloseable { - - // IDS-6072 - /* - * java.lang.RuntimeException: Error in Python interpreter: - * Type: - * Value: cannot convert a Python 'list' to a Java 'java.util.List' - * Line: - * Namespace: - * File: - * at org.jpy.PyLib.callAndReturnValue(Native Method) - * at org.jpy.PyProxyHandler.invoke(PyProxyHandler.java:80) - * at io.deephaven.python.$Proxy7.extract_expression_names(Unknown Source) - * at io.deephaven.python.ExtractFormulaNamesTest.extract_expression_names(ExtractFormulaNamesTest.java:26) - */ - //List extract_expression_names(String pythonExpression); - - /** - * Extract names from an expression string - * - * @param pythonExpression the python expression, of the form - * @return a sorted list of the named elements from - */ - PyObject extract_expression_names(String pythonExpression); - - @Override - void close(); - - /** - * A helper to convert the returned results. - * - *

A workaround until IDS-6072 is addressed. - * - * @param object the python object, as returned from {@link #extract_expression_names(String)} - * @return the list - */ - static List convertToList(PyObject object) { - final List list = object.asList(); - for (PyObject pyObject : list) { - if (!pyObject.isString()) { - throw new ClassCastException("Expecting all items in the list to be python strings"); - } - } - return list.stream().map(PyObject::str).collect(Collectors.toList()); - } -} diff --git a/DB/src/main/java/io/deephaven/python/ASTHelperSetup.java b/DB/src/main/java/io/deephaven/python/ASTHelperSetup.java deleted file mode 100644 index 406420c0b7f..00000000000 --- a/DB/src/main/java/io/deephaven/python/ASTHelperSetup.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.deephaven.python; - -import java.lang.invoke.MethodHandles; -import java.util.Objects; -import org.jpy.PyLibInitializer; - -public class ASTHelperSetup { - - private static final String NAME = "ast_helper"; - private static final String RESOURCE = NAME + ".py"; - - private static volatile ASTHelper instance; - - public static ASTHelper create() { - if (!PyLibInitializer.isPyLibInitialized()) { - throw new IllegalStateException("PyLib not initialized"); - } - return PyModuleFromResource - .load(MethodHandles.lookup().lookupClass(), NAME, RESOURCE) - .createProxy(ASTHelper.class); - } - - public static ASTHelper getInstance() { - ASTHelper local; - if ((local = instance) == null) { - synchronized (ASTHelperSetup.class) { - if ((local = instance) == null) { - local = (instance = Objects.requireNonNull(create())); - } - } - } - return local; - } -} diff --git a/Integrations/python/deephaven/java/__init__.py b/Integrations/python/deephaven/java/__init__.py new file mode 100644 index 00000000000..1b5f20b2a4d --- /dev/null +++ b/Integrations/python/deephaven/java/__init__.py @@ -0,0 +1,2 @@ +from .integration import * +from .primitives import * diff --git a/Integrations/python/deephaven/java/integration.py b/Integrations/python/deephaven/java/integration.py new file mode 100644 index 00000000000..4f152d332a2 --- /dev/null +++ b/Integrations/python/deephaven/java/integration.py @@ -0,0 +1,29 @@ +import jpy + +from . import primitives + +if jpy.has_jvm(): + + __DeephavenCompatibleFunction__ = jpy.get_type('io.deephaven.db.v2.select.python.DeephavenCompatibleFunction') + __FormulaColumnPython__ = jpy.get_type('io.deephaven.db.v2.select.python.FormulaColumnPython') + __ConditionFilterPython__ = jpy.get_type('io.deephaven.db.v2.select.python.ConditionFilterPython') + + + def formula(func, *, name, column_names, return_type=None, is_vectorized=None): + if hasattr(func, '__dh_return_type__'): + return_type = return_type if return_type is not None else func.__dh_return_type__ + if hasattr(func, '__dh_is_vectorized__'): + is_vectorized = is_vectorized if is_vectorized is not None else func.__dh_is_vectorized__ + if return_type is None: + raise TypeError("Must provide return_type") + if is_vectorized is None: + raise TypeError("Must provide is_vectorized") + return __FormulaColumnPython__.create(name, __DeephavenCompatibleFunction__.create(func, return_type, column_names, + is_vectorized)) + + + def filter(func, *, column_names, is_vectorized): + if hasattr(func, '__dh_is_vectorized__'): + is_vectorized = is_vectorized if is_vectorized is not None else func.__dh_is_vectorized__ + return __ConditionFilterPython__.create( + __DeephavenCompatibleFunction__.create(func, primitives.boolean, column_names, is_vectorized)) diff --git a/Integrations/python/deephaven/java/primitives.py b/Integrations/python/deephaven/java/primitives.py new file mode 100644 index 00000000000..dd81995de82 --- /dev/null +++ b/Integrations/python/deephaven/java/primitives.py @@ -0,0 +1,16 @@ +import jpy + +if jpy.has_jvm(): + boolean = jpy.get_type('boolean') + byte = jpy.get_type('byte') + char = jpy.get_type('char') + short = jpy.get_type('short') + int = jpy.get_type('int') + long = jpy.get_type('long') + float = jpy.get_type('float') + double = jpy.get_type('double') + + __primitives = frozenset([ boolean, byte, char, short, int, long, float, double ]) + + def is_primitive_type(type): + return type in __primitives diff --git a/Integrations/python/deephaven/numba/__init__.py b/Integrations/python/deephaven/numba/__init__.py new file mode 100644 index 00000000000..4626203fb56 --- /dev/null +++ b/Integrations/python/deephaven/numba/__init__.py @@ -0,0 +1 @@ +from .integration import * \ No newline at end of file diff --git a/Integrations/python/deephaven/numba/integration.py b/Integrations/python/deephaven/numba/integration.py new file mode 100644 index 00000000000..b96472bc877 --- /dev/null +++ b/Integrations/python/deephaven/numba/integration.py @@ -0,0 +1,110 @@ +import inspect +import numba +import numpy + +from .. import java +import jpy + +if jpy.has_jvm(): + + # note: these numba "types" are not basic python types + __java_type_to_numba_type = { + java.boolean: numba.boolean, + java.byte: numba.byte, + java.short: numba.int16, + java.int: numba.int32, + java.long: numba.int64, + java.float: numba.float32, + java.double: numba.float64 + } + + # note: these numba "types" are not basic python types + __numpy_type_to_numba_type = { + numpy.bool_: numba.boolean, + numpy.int8: numba.byte, + numpy.int16: numba.int16, + numpy.int32: numba.int32, + numpy.int64: numba.int64, + numpy.float32: numba.float32, + numpy.float64: numba.float64 + } + + + def is_numpy_type(x): + return isinstance(x, type) and issubclass(x, numpy.generic) + + + def get_numba_type(x): + if java.is_primitive_type(x): + return_type = __java_type_to_numba_type[x] + if not return_type: + raise TypeError(f"Unable to find numba type for java primitive {x}") + return return_type + + if is_numpy_type(x): + return_type = __numpy_type_to_numba_type[x] + if not return_type: + raise TypeError(f"Unable to find numba type for numpy type {x}") + return return_type + + return x + + + def create_numba_signature(signature: inspect.Signature) -> numba.core.typing.templates.Signature: + ''' + Uses annotation hints to create a numba signature. + + :param signature: the signature + :return: the numba signature + ''' + arg_annotations = [get_numba_type(param.annotation) for param in signature.parameters.values()] + # this creates a "strongly-typed" signature like numba.int32(numba.int64,numba.float64) + return get_numba_type(signature.return_annotation)(*arg_annotations) + + + def vectorize(_func=None, *, identity=None, nopython=True, target='cpu', cache=False): + ''' + Returns a new Deephaven-compatible function (or decorator) based off of + numba.vectorize. + + :param _func: the function + :return: the Deephaven-compatible function + ''' + + def inner_decorator(func): + signature = inspect.signature(func) + numba_signature = create_numba_signature(signature) + numba_vectorized = numba.vectorize([numba_signature], identity=identity, nopython=nopython, target=target, + cache=cache)(func) + numba_vectorized.__dh_return_type__ = signature.return_annotation + numba_vectorized.__dh_is_vectorized__ = True + return numba_vectorized + + if _func is None: + return inner_decorator + else: + return inner_decorator(_func) + + + def njit(_func=None, *, nopython=True, cache=False, parallel=False, fastmath=False): + ''' + Returns a new Deephaven-compatible function (or decorator) based off of + numba.jit. + + :param _func: the function + :return: the Deephaven-compatible function + ''' + + def inner_decorator(func): + signature = inspect.signature(func) + numba_signature = create_numba_signature(signature) + numba_jitted = numba.jit(numba_signature, nopython=nopython, cache=cache, parallel=parallel, fastmath=fastmath)( + func) + numba_jitted.__dh_return_type__ = signature.return_annotation + numba_jitted.__dh_is_vectorized__ = False + return numba_jitted + + if _func is None: + return inner_decorator + else: + return inner_decorator(_func) diff --git a/py/jpy/src/main/java/org/jpy/Assignment.java b/py/jpy/src/main/java/org/jpy/Assignment.java new file mode 100644 index 00000000000..509dcbc3076 --- /dev/null +++ b/py/jpy/src/main/java/org/jpy/Assignment.java @@ -0,0 +1,42 @@ +package org.jpy; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +class Assignment { + + private static final Map, Class> boxedToPrimitive; + + static { + boxedToPrimitive = new HashMap<>(); + boxedToPrimitive.put(Boolean.class, boolean.class); + boxedToPrimitive.put(Byte.class, byte.class); + boxedToPrimitive.put(Character.class, char.class); + boxedToPrimitive.put(Short.class, short.class); + boxedToPrimitive.put(Integer.class, int.class); + boxedToPrimitive.put(Long.class, long.class); + boxedToPrimitive.put(Float.class, float.class); + boxedToPrimitive.put(Double.class, double.class); + } + + static Optional> getUnboxedType(Class clazz) { + return Optional.ofNullable(boxedToPrimitive.get(clazz)); + } + + static boolean isAssignableFrom(Class signatureType, Object instance) { + return isAssignableFrom(signatureType, instance.getClass()); + } + + static boolean isAssignableFrom(Class signatureType, Class instanceType) { + if (signatureType.isAssignableFrom(instanceType)) { + return true; + } + if (signatureType.isPrimitive()) { + return getUnboxedType(instanceType) + .filter(signatureType::isAssignableFrom) + .isPresent(); + } + return false; + } +} diff --git a/py/jpy/src/main/java/org/jpy/PyObject.java b/py/jpy/src/main/java/org/jpy/PyObject.java index 89af1924d27..e0715ec7477 100644 --- a/py/jpy/src/main/java/org/jpy/PyObject.java +++ b/py/jpy/src/main/java/org/jpy/PyObject.java @@ -462,7 +462,7 @@ public T call(Class returnType, String name, Class[] paramTypes, Objec // Let's be defensive here right now - we can loosen this in the future if necessary. Objects.requireNonNull(paramTypes[i], "paramTypes items must be non null"); Objects.requireNonNull(args[i], "args items must be non null"); - if (!paramTypes[i].isAssignableFrom(args[i].getClass())) { + if (!Assignment.isAssignableFrom(paramTypes[i], args[i])) { throw new IllegalArgumentException(String.format( "Argument %d of type '%s' is not assignable to type '%s'", i, diff --git a/py/jpy/src/main/java/org/jpy/PyProxyHandler.java b/py/jpy/src/main/java/org/jpy/PyProxyHandler.java index 7f7a8d71b01..08f8a264163 100644 --- a/py/jpy/src/main/java/org/jpy/PyProxyHandler.java +++ b/py/jpy/src/main/java/org/jpy/PyProxyHandler.java @@ -16,6 +16,9 @@ package org.jpy; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodHandles.Lookup; +import java.lang.reflect.Constructor; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.util.Arrays; @@ -65,6 +68,33 @@ public PyProxyHandler(PyObject pyObject, PyLib.CallableKind callableKind) { public Object invoke(Object proxyObject, Method method, Object[] args) throws Throwable { //assertPythonRuns(); // todo: get rid of this check to remove a call down into JNI? + if (method.isDefault()) { + // This allows our proxy-able interfaces to define default methods. + // Note: in this current implementation, defaults methods will always take precedence. + final Class declaringClass = method.getDeclaringClass(); + + // https://blog.jooq.org/2018/03/28/correct-reflective-access-to-interface-default-methods-in-java-8-9-10/ + + // note: the following throws an IllegalAccessException of the form no private access for invokespecial + //return MethodHandles.lookup() + // .in(declaringClass) + // .unreflectSpecial(method, declaringClass) + // .bindTo(proxyObject) + // .invokeWithArguments(args); + + // Unfortunately, the following doesn't work w/ Java 9+. There should be some new api + // methods that work w/ Java 9+ though. (MethodHandles#privateLookupIn) + final Constructor constructor = Lookup.class + .getDeclaredConstructor(Class.class); + constructor.setAccessible(true); + return constructor + .newInstance(declaringClass) + .in(declaringClass) + .unreflectSpecial(method, declaringClass) + .bindTo(proxyObject) + .invokeWithArguments(args); + } + final long pointer = this.pyObject.getPointer(); if ((PyLib.Diag.getFlags() & PyLib.Diag.F_METH) != 0) { From 23a31a90079bd7fbcf40c3c25c14363c8f1ad3a7 Mon Sep 17 00:00:00 2001 From: Mike Blaszczak Date: Mon, 24 May 2021 13:37:17 -0700 Subject: [PATCH 07/10] Deploy example scripts and data (#562) * port samples from old core project * get the right file set; fix URL to public repository * revise sample script to use git * round of changes * connect to correct github path with published files * Update samples/sample_script.sh Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com> * fixes from Chip feedback * copyedit; remove target version option from version command * add git pull in download when repo exists Co-authored-by: Mike Blaszczak Co-authored-by: Chip Kent <5250374+chipkent@users.noreply.github.com> --- .gitignore | 1 + samples/Dockerfile | 8 +++ samples/sample_script.sh | 116 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 samples/Dockerfile create mode 100755 samples/sample_script.sh diff --git a/.gitignore b/.gitignore index d15bd8e5c73..b4628d1d0de 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ out/ *.ipr *.iws *.iml +data/ tmp/ target/ IRIS_GRADLE_VERSION diff --git a/samples/Dockerfile b/samples/Dockerfile new file mode 100644 index 00000000000..9170da193c9 --- /dev/null +++ b/samples/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine:3.7 +RUN apk add --no-cache git git-lfs + +VOLUME [ "/data" ] +ENTRYPOINT [ "/sample_script.sh" ] + +COPY sample_script.sh . + diff --git a/samples/sample_script.sh b/samples/sample_script.sh new file mode 100755 index 00000000000..858f5581da7 --- /dev/null +++ b/samples/sample_script.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env sh + +# set -xv + +# show usage and quit +function usage() +{ + printf "usage: %s \n" $(basename $0) >&2 + printf "\n" >&2 + printf " commands are:\n" + printf " download [] - downloads and mounts all example data\n" >&2 + printf " gets latest version, unless supplied\n" >&2 + printf " remove - removes all example data\n" >&2 + printf " version - shows current the version\n" >&2 + printf " versions - list available versions\n" >&2 + exit 2 +} + + +# complain and quit +function fail_out() +{ + printf "Failed! %s\n" "$@" >&2 + exit 2 +} + + +# check that we have the expected enlistment directory; download it if not +function ensure_enlistment() +{ + if [ ! -d $target_path/.git ]; then + printf "no examples collection at %s; dowloading ..." $target_path >&2 + do_download + fi +} + + +# clone the git repo, don't report progress but DO report errors +function do_download() +{ + if [ -d $target_path/.git ]; then + printf "examples collection already exists at %s\n" $target_path >&2 + git pull || fail_out "Couldn't update existing collection" + else + git clone --quiet $git_root_url $target_path || fail_out "Couldn't clone examples repository" + printf "examples downloaded to $target_path\n" + fi + + if [ ! -z "$1" ]; then + do_checkout_version "$1" + else + cd $target_path + do_checkout_version `git describe --tags $(git rev-list --tags --max-count=1)` + fi +} + + +# remove the enlistment directory +function do_remove() +{ + [ -d $target_path ] || fail_out "Couldn't find $target_path$root_path" + rm -rf $target_path + printf "$target_path removed\n" +} + + +# list all the tags we know +function do_list_versions() +{ + cd $target_path + printf "local versions follow:\n" + git tag -n + printf "remote versions follow:\n" + git ls-remote --tags $git_root_url | grep -v "{}" | awk '{print $2}' | sed 's/refs\/tags\///' + printf "Version listed\n" +} + + +# switch version to something different +function do_checkout_version() +{ + cd $target_path + printf "checkout out version %s ...\n" "$1" + git -c advice.detachedHead=false checkout "$1" || fail_out "Couldn't change versions" + printf "set to version %s\n" "$1" +} + + +##### +# set up the source and target info +git_root_url="git://github.com/deephaven/examples.git" + +target_path="/data/examples" + +# figure out command and dispatch ... +case "$1" in + download) + do_download "$2" + ;; + version) + ensure_enlistment + cd $target_path + git describe + ;; + versions) + ensure_enlistment + do_list_versions + ;; + remove) + do_remove + ;; + *) + printf "Unknown command '%s'\n" $1 >&2 + usage + ;; +esac From 8f88e2b5c9ef0cf02cb2ffa5a9910d58dfe09308 Mon Sep 17 00:00:00 2001 From: Cristian Ferretti <37232625+jcferretti@users.noreply.github.com> Date: Mon, 24 May 2021 18:12:47 -0400 Subject: [PATCH 08/10] Fix integer overflow in SortedRanges.subRangeByPos. Fixes #664 (#665) Co-authored-by: Cristian Ferretti --- .../io/deephaven/db/v2/utils/sortedranges/SortedRanges.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java b/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java index f367c9b2448..62253a2f13e 100644 --- a/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java +++ b/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java @@ -1040,7 +1040,7 @@ public final SortedRanges subRangesByPos(final long startPosIn, final long endPo // We don't want to do two passes, we allocated an array big enough instead. final boolean brokenInitialRange = startPos < pos; int ansLen = count - i + (brokenInitialRange ? 2 : 1); - ansLen = Math.min(ansLen, (int) (inputRangeSpan + 1)); + ansLen = (int) Math.min(ansLen, (inputRangeSpan + 1)); final SortedRanges ans = makeMyTypeAndOffset(ansLen); ans.count = 0; ans.cardinality = 0; From 95279db5ef78fcd64259bf6a1b5341377f3461b3 Mon Sep 17 00:00:00 2001 From: Cristian Ferretti <37232625+jcferretti@users.noreply.github.com> Date: Tue, 25 May 2021 09:24:29 -0400 Subject: [PATCH 09/10] Fix integer overflow in SortedRanges.invertOnNew. Fixes #666. (#667) * Fix integer overflow in SortedRanges.invertOnNew. Fixes #666. * Fix another instance of an int pos that should have been long. Co-authored-by: Cristian Ferretti --- .../v2/utils/sortedranges/SortedRanges.java | 4 +-- .../utils/TestIncrementalReleaseFilter.java | 30 +++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java b/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java index 62253a2f13e..9c187f8cb8c 100644 --- a/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java +++ b/DB/src/main/java/io/deephaven/db/v2/utils/sortedranges/SortedRanges.java @@ -2191,7 +2191,7 @@ static boolean removeLegacy(final MutableObject sarOut, final Inde public final TreeIndexImpl invertRangeOnNew(final long start, final long end, final long maxPosition) { final long packedStart = pack(start); int i = 0; - int pos = 0; + long pos = 0; long data = packedGet(i); boolean neg = false; long pendingStart = -1; @@ -2260,7 +2260,7 @@ public final boolean invertOnNew( long end = rit.currentRangeEnd(); long packedStart = pack(start); int i = 0; - int pos = 0; + long pos = 0; long data = packedGet(i); boolean neg = false; long pendingStart = -1; diff --git a/DB/src/test/java/io/deephaven/db/v2/utils/TestIncrementalReleaseFilter.java b/DB/src/test/java/io/deephaven/db/v2/utils/TestIncrementalReleaseFilter.java index b1dc33b64eb..dbcdd8f6ea6 100644 --- a/DB/src/test/java/io/deephaven/db/v2/utils/TestIncrementalReleaseFilter.java +++ b/DB/src/test/java/io/deephaven/db/v2/utils/TestIncrementalReleaseFilter.java @@ -14,10 +14,12 @@ import io.deephaven.db.v2.select.IncrementalReleaseFilter; import org.junit.experimental.categories.Category; -import static io.deephaven.db.v2.TstUtils.getTable; -import static io.deephaven.db.v2.TstUtils.initColumnInfos; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static io.deephaven.db.v2.TstUtils.assertTableEquals; -@Category(OutOfBandTest.class) public class TestIncrementalReleaseFilter extends LiveTableTestCase { public void testSimple() { final Table source = TableTools.newTable(TableTools.intCol("Sentinel", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)); @@ -37,6 +39,28 @@ public void testSimple() { } } + public void testBigTable() { + final Table sourcePart = TableTools.emptyTable(1_000_000_000L); + final List

sourceParts = IntStream.range(0, 20).mapToObj(x -> sourcePart).collect(Collectors.toList()); + final Table source = TableTools.merge(sourceParts); + TableTools.show(source); + + final IncrementalReleaseFilter incrementalReleaseFilter = new IncrementalReleaseFilter(2, 10_000_000); + final Table filtered = source.where(incrementalReleaseFilter); + final Table flattened = filtered.flatten(); + + assertEquals(2, filtered.size()); + + int cycles = 0; + while (filtered.size() < source.size()) { + LiveTableMonitor.DEFAULT.runWithinUnitTestCycle(incrementalReleaseFilter::refresh); + cycles++; + } + assertTableEquals(source, filtered); + assertTableEquals(flattened, filtered); + System.out.println("Cycles: " + cycles); + } + static public T sleepValue(long duration, T retVal) { final Object blech = new Object(); //noinspection SynchronizationOnLocalVariableOrMethodParameter From 0142a72589f2cfa7a22697891b164433122627b6 Mon Sep 17 00:00:00 2001 From: Devin Smith Date: Tue, 25 May 2021 10:31:08 -0500 Subject: [PATCH 10/10] Check documentation labels, fixes #627 (#671) --- .github/scripts/check-doc-labels.sh | 20 ++++++++++++++++++++ .github/workflows/label-check-ci.yml | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100755 .github/scripts/check-doc-labels.sh create mode 100644 .github/workflows/label-check-ci.yml diff --git a/.github/scripts/check-doc-labels.sh b/.github/scripts/check-doc-labels.sh new file mode 100755 index 00000000000..ab69ea602e1 --- /dev/null +++ b/.github/scripts/check-doc-labels.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset + +if [ "${HAS_DocumentationNeeded}" == "true" ] ; then + if [ "${HAS_NoDocumentationNeeded}" == "true" ] ; then + >&2 echo "Conflicting documentation requirements" + exit 1 + fi + exit 0 +fi + +if [ "${HAS_NoDocumentationNeeded}" == "true" ] ; then + exit 0 +fi + +>&2 echo "No documentation requirements found" +exit 1 diff --git a/.github/workflows/label-check-ci.yml b/.github/workflows/label-check-ci.yml new file mode 100644 index 00000000000..efb054a3583 --- /dev/null +++ b/.github/workflows/label-check-ci.yml @@ -0,0 +1,22 @@ +name: Label Check CI + +on: + pull_request: + # This set of types only causes this workflow to run once when it is opened, or when the labels + # change. For example, it won't be (and doesn't need to be) retriggered when new commits are + # pushed. + types: [ labeled, unlabeled, opened ] + branches: [ main ] + +jobs: + doc-labels: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Check Documentation Labels + env: + HAS_DocumentationNeeded: ${{ contains(github.event.pull_request.labels.*.name, 'DocumentationNeeded') }} + HAS_NoDocumentationNeeded: ${{ contains(github.event.pull_request.labels.*.name, 'NoDocumentationNeeded') }} + run: .github/scripts/check-doc-labels.sh