From 1e3686b998423b8944d3ba17bea4a5663e0d0145 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 26 Jan 2022 17:48:28 -0600 Subject: [PATCH 1/6] draft of dataframe fetch from js --- .../src/main/java/io/deephaven/ide/shared/IdeSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/client-api/src/main/java/io/deephaven/ide/shared/IdeSession.java b/web/client-api/src/main/java/io/deephaven/ide/shared/IdeSession.java index 6f17b0f71f3..2f339f64661 100644 --- a/web/client-api/src/main/java/io/deephaven/ide/shared/IdeSession.java +++ b/web/client-api/src/main/java/io/deephaven/ide/shared/IdeSession.java @@ -103,7 +103,7 @@ public Promise getFigure(String name) { return getVariableDefinition(name, JsVariableChanges.FIGURE).then(connection::getFigure); } - public Promise getObject(JsPropertyMap definitionObject) { + public Promise getObject(JsPropertyMap definitionObject) { if (definitionObject instanceof JsVariableDefinition) { return connection.getObject((JsVariableDefinition) definitionObject); } From 7009425ad7d8545f50ad84e5f90c8c5166365aac Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 28 Jan 2022 10:08:03 -0600 Subject: [PATCH 2/6] Python wiring to convert dataframes to tables, js wiring to expect this plugin --- Integrations/python/deephaven/pandas/__init__.py | 7 +++++++ .../python/deephaven/pandas/pandas_as_table.py | 16 ++++++++++++++++ Integrations/python/setup.cfg | 4 ++++ Integrations/python/setup.py | 10 ++++++++-- .../client/api/console/JsVariableChanges.java | 2 +- 5 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 Integrations/python/deephaven/pandas/__init__.py create mode 100644 Integrations/python/deephaven/pandas/pandas_as_table.py diff --git a/Integrations/python/deephaven/pandas/__init__.py b/Integrations/python/deephaven/pandas/__init__.py new file mode 100644 index 00000000000..3e2b1107796 --- /dev/null +++ b/Integrations/python/deephaven/pandas/__init__.py @@ -0,0 +1,7 @@ +from deephaven.plugin import Registration +from . import pandas_as_table + +class PandasPluginRegistration(Registration): + @classmethod + def register_into(clscls, callback: Registration.Callback) -> None: + callback.register(pandas_as_table.PandasDataFrameSerializer) \ No newline at end of file diff --git a/Integrations/python/deephaven/pandas/pandas_as_table.py b/Integrations/python/deephaven/pandas/pandas_as_table.py new file mode 100644 index 00000000000..97948a93f39 --- /dev/null +++ b/Integrations/python/deephaven/pandas/pandas_as_table.py @@ -0,0 +1,16 @@ +from deephaven.python_to_java import dataFrameToTable +from deephaven.plugin.object import Exporter, ObjectType +from pandas import DataFrame + +NAME = "pandas.DataFrame" +class PandasDataFrameSerializer(ObjectType): + @property + def name(self) -> str: + return NAME + + def is_type(self, object) -> bool: + return isinstance(object, DataFrame) + + def to_bytes(self, exporter: Exporter, data_frame: DataFrame): + exporter.reference(dataFrameToTable(data_frame)) + return b'' diff --git a/Integrations/python/setup.cfg b/Integrations/python/setup.cfg index 3c6e79cf31d..8d118e2942d 100644 --- a/Integrations/python/setup.cfg +++ b/Integrations/python/setup.cfg @@ -1,2 +1,6 @@ [bdist_wheel] universal=1 + +[options.entry_points] +deephaven.plugin = + registration_cls = deephaven.pandas:PandasPluginRegistration \ No newline at end of file diff --git a/Integrations/python/setup.py b/Integrations/python/setup.py index e8452fb644b..ec016da01b2 100644 --- a/Integrations/python/setup.py +++ b/Integrations/python/setup.py @@ -53,6 +53,12 @@ def normalize_version(ver): keywords='Deephaven Development', packages=find_packages(exclude=['docs', 'test']), install_requires=['deephaven-jpy=={}'.format(__normalized_version__), - 'numpy', 'dill>=0.2.8', 'wrapt', 'pandas', 'numba;python_version>"3.0"', - 'enum34;python_version<"3.4"'], + 'deephaven-plugin', + 'numpy', + 'dill>=0.2.8', + 'wrapt', + 'pandas', + 'numba;python_version>"3.0"', + 'enum34;python_version<"3.4"' + ], ) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/console/JsVariableChanges.java b/web/client-api/src/main/java/io/deephaven/web/client/api/console/JsVariableChanges.java index 547d6e83f10..03aae4e6507 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/console/JsVariableChanges.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/console/JsVariableChanges.java @@ -14,7 +14,7 @@ public class JsVariableChanges { TABLEMAP = "TableMap", FIGURE = "Figure", OTHERWIDGET = "OtherWidget", - PANDAS = "Pandas"; + PANDAS = "pandas.DataFrame"; private final JsVariableDefinition[] created; private final JsVariableDefinition[] updated; From 8ab67f89ccdfad7a6a41ba877c45fc45c9bdf6d5 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 28 Jan 2022 14:07:18 -0600 Subject: [PATCH 3/6] addit cleanup --- Integrations/python/deephaven/pandas/pandas_as_table.py | 2 ++ Integrations/python/setup.cfg | 2 +- .../java/io/deephaven/web/client/api/WorkerConnection.java | 7 +++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Integrations/python/deephaven/pandas/pandas_as_table.py b/Integrations/python/deephaven/pandas/pandas_as_table.py index 97948a93f39..588d93fc6ab 100644 --- a/Integrations/python/deephaven/pandas/pandas_as_table.py +++ b/Integrations/python/deephaven/pandas/pandas_as_table.py @@ -3,6 +3,8 @@ from pandas import DataFrame NAME = "pandas.DataFrame" + + class PandasDataFrameSerializer(ObjectType): @property def name(self) -> str: diff --git a/Integrations/python/setup.cfg b/Integrations/python/setup.cfg index 8d118e2942d..36fb892d8d5 100644 --- a/Integrations/python/setup.cfg +++ b/Integrations/python/setup.cfg @@ -3,4 +3,4 @@ universal=1 [options.entry_points] deephaven.plugin = - registration_cls = deephaven.pandas:PandasPluginRegistration \ No newline at end of file + registration_cls = deephaven.pandas:PandasPluginRegistration diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java b/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java index b5c9b46275b..c2b43bc6ee5 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java @@ -699,10 +699,13 @@ public Promise getPandas(JsVariableDefinition varDef) { @SuppressWarnings({"unchecked", "rawtypes"}) public Promise getObject(JsVariableDefinition definition) { - if (definition.getType().equals(JsVariableChanges.TABLE)) { + if (JsVariableChanges.TABLE.equals(definition.getType())) { return (Promise) getTable(definition, null); - } else if (definition.getType().equals(JsVariableChanges.FIGURE)) { + } else if (JsVariableChanges.FIGURE.equals(definition.getType())) { return (Promise) getFigure(definition); + } else if (JsVariableChanges.PANDAS.equals(definition.getType())) { + return getWidget(definition) + .then(widget -> widget.getExportedObjects()[0].fetch()); } else { return (Promise) getWidget(definition); } From 6cc08849219eb1f93ddae97b56de4ff9fb61e6b8 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 4 Feb 2022 16:03:33 -0600 Subject: [PATCH 4/6] cleanup --- .../python/deephaven/pandas/__init__.py | 2 +- .../web/client/api/WorkerConnection.java | 26 +++---------------- .../api/widget/JsWidgetExportedObject.java | 2 +- 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/Integrations/python/deephaven/pandas/__init__.py b/Integrations/python/deephaven/pandas/__init__.py index 3e2b1107796..f1e1cb856d8 100644 --- a/Integrations/python/deephaven/pandas/__init__.py +++ b/Integrations/python/deephaven/pandas/__init__.py @@ -4,4 +4,4 @@ class PandasPluginRegistration(Registration): @classmethod def register_into(clscls, callback: Registration.Callback) -> None: - callback.register(pandas_as_table.PandasDataFrameSerializer) \ No newline at end of file + callback.register(pandas_as_table.PandasDataFrameSerializer) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java b/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java index c2b43bc6ee5..b5b15759409 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/WorkerConnection.java @@ -680,34 +680,16 @@ public Promise getTable(JsVariableDefinition varDef, @Nullable Boolean }); } - public Promise getPandas(JsVariableDefinition varDef) { - return whenServerReady("get a pandas table").then(serve -> { - JsLog.debug("innerGetPandasTable", varDef.getTitle(), " started"); - return newState(info, - (c, cts, metadata) -> { - JsLog.debug("performing fetch for ", varDef.getTitle(), " / ", cts, - " (" + LazyString.of(cts::getHandle), ")"); - throw new UnsupportedOperationException("getPandas"); - - }, "fetch pandas table " + varDef.getTitle()).then(cts -> { - JsLog.debug("innerGetPandasTable", varDef.getTitle(), " succeeded ", cts); - JsTable table = new JsTable(this, cts); - return Promise.resolve(table); - }); - }); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public Promise getObject(JsVariableDefinition definition) { + public Promise getObject(JsVariableDefinition definition) { if (JsVariableChanges.TABLE.equals(definition.getType())) { - return (Promise) getTable(definition, null); + return getTable(definition, null); } else if (JsVariableChanges.FIGURE.equals(definition.getType())) { - return (Promise) getFigure(definition); + return getFigure(definition); } else if (JsVariableChanges.PANDAS.equals(definition.getType())) { return getWidget(definition) .then(widget -> widget.getExportedObjects()[0].fetch()); } else { - return (Promise) getWidget(definition); + return getWidget(definition); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/JsWidgetExportedObject.java b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/JsWidgetExportedObject.java index f784acfc727..a42f6c1b93f 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/widget/JsWidgetExportedObject.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/widget/JsWidgetExportedObject.java @@ -28,7 +28,7 @@ public String getType() { } @JsMethod - public Promise fetch() { + public Promise fetch() { if (getType().equals(JsVariableChanges.TABLE)) { return Callbacks.grpcUnaryPromise(c -> { connection.tableServiceClient().getExportedTableCreationResponse(ticket.getTicket(), From 7e7852dda7e8d9a16ecf093db6b7ddb9b715df83 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 27 Apr 2022 11:09:46 -0500 Subject: [PATCH 5/6] rebase damage --- Integrations/python/setup.cfg | 4 ---- Integrations/python/setup.py | 3 +-- .../server/deephaven/pandasplugin}/__init__.py | 0 .../deephaven/pandasplugin}/pandas_as_table.py | 4 ++-- py/server/setup.py | 14 ++++++++++++-- 5 files changed, 15 insertions(+), 10 deletions(-) rename {Integrations/python/deephaven/pandas => py/server/deephaven/pandasplugin}/__init__.py (100%) rename {Integrations/python/deephaven/pandas => py/server/deephaven/pandasplugin}/pandas_as_table.py (77%) diff --git a/Integrations/python/setup.cfg b/Integrations/python/setup.cfg index 36fb892d8d5..3c6e79cf31d 100644 --- a/Integrations/python/setup.cfg +++ b/Integrations/python/setup.cfg @@ -1,6 +1,2 @@ [bdist_wheel] universal=1 - -[options.entry_points] -deephaven.plugin = - registration_cls = deephaven.pandas:PandasPluginRegistration diff --git a/Integrations/python/setup.py b/Integrations/python/setup.py index ec016da01b2..8b6bf1422bb 100644 --- a/Integrations/python/setup.py +++ b/Integrations/python/setup.py @@ -53,12 +53,11 @@ def normalize_version(ver): keywords='Deephaven Development', packages=find_packages(exclude=['docs', 'test']), install_requires=['deephaven-jpy=={}'.format(__normalized_version__), - 'deephaven-plugin', 'numpy', 'dill>=0.2.8', 'wrapt', 'pandas', 'numba;python_version>"3.0"', - 'enum34;python_version<"3.4"' + 'enum34;python_version<"3.4"', ], ) diff --git a/Integrations/python/deephaven/pandas/__init__.py b/py/server/deephaven/pandasplugin/__init__.py similarity index 100% rename from Integrations/python/deephaven/pandas/__init__.py rename to py/server/deephaven/pandasplugin/__init__.py diff --git a/Integrations/python/deephaven/pandas/pandas_as_table.py b/py/server/deephaven/pandasplugin/pandas_as_table.py similarity index 77% rename from Integrations/python/deephaven/pandas/pandas_as_table.py rename to py/server/deephaven/pandasplugin/pandas_as_table.py index 588d93fc6ab..a84f00f849b 100644 --- a/Integrations/python/deephaven/pandas/pandas_as_table.py +++ b/py/server/deephaven/pandasplugin/pandas_as_table.py @@ -1,4 +1,4 @@ -from deephaven.python_to_java import dataFrameToTable +from deephaven.pandas import to_table from deephaven.plugin.object import Exporter, ObjectType from pandas import DataFrame @@ -14,5 +14,5 @@ def is_type(self, object) -> bool: return isinstance(object, DataFrame) def to_bytes(self, exporter: Exporter, data_frame: DataFrame): - exporter.reference(dataFrameToTable(data_frame)) + exporter.reference(to_table(data_frame)) return b'' diff --git a/py/server/setup.py b/py/server/setup.py index 7015760b5fc..6b46e06f316 100644 --- a/py/server/setup.py +++ b/py/server/setup.py @@ -44,6 +44,16 @@ def normalize_version(version): ], keywords='Deephaven Development', install_requires=['deephaven-jpy=={}'.format(__normalized_version__), - 'numpy', 'dill>=0.2.8', 'wrapt', 'pandas', 'numba;python_version>"3.0"', - 'enum34;python_version<"3.4"'], + 'deephaven-plugin', + 'numpy', + 'dill>=0.2.8', + 'wrapt', + 'pandas', + 'numba;python_version>"3.0"', + 'enum34;python_version<"3.4"' + ], + entry_points=''' + [deephaven.plugin] + registration_cls = deephaven.pandasplugin:PandasPluginRegistration + ''' ) From 3be79670ca6271c4a4a18a308a69155b5bba46c4 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 27 Apr 2022 14:39:51 -0500 Subject: [PATCH 6/6] Change configuration to use a dict --- py/server/setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/py/server/setup.py b/py/server/setup.py index 6b46e06f316..d0e40300e07 100644 --- a/py/server/setup.py +++ b/py/server/setup.py @@ -52,8 +52,7 @@ def normalize_version(version): 'numba;python_version>"3.0"', 'enum34;python_version<"3.4"' ], - entry_points=''' - [deephaven.plugin] - registration_cls = deephaven.pandasplugin:PandasPluginRegistration - ''' + entry_points={ + 'deephaven.plugin': ['registration_cls = deephaven.pandasplugin:PandasPluginRegistration'] + }, )