Sync delta-spark requirements (#160)

* Update setup.py * Update releases notes Signed-off-by: Nok <nok.lam.chan@quantumblack.com> --------- Signed-off-by: Nok <nok.lam.chan@quantumblack.com>
kedro-org · Apr 6, 2023 · 31476ce · 31476ce
1 parent e9fbbc7
commit 31476ce
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/kedro-datasets/RELEASE.md b/kedro-datasets/RELEASE.md
@@ -5,6 +5,7 @@
 * Added SQLAlchemy 2.0 support (and dropped support for versions below 1.4).
 
 ## Bug fixes and other changes
+* Relaxed `delta-spark` upper bound to allow compatibility with Spark 3.1.x and 3.2.x.
 
 # Release 1.2.0:
 

diff --git a/kedro-datasets/setup.py b/kedro-datasets/setup.py
@@ -76,7 +76,7 @@ def _collect_requirements(requires):
     "spark.SparkDataSet": [SPARK, HDFS, S3FS],
     "spark.SparkHiveDataSet": [SPARK, HDFS, S3FS],
     "spark.SparkJDBCDataSet": [SPARK, HDFS, S3FS],
-    "spark.DeltaTableDataSet": [SPARK, HDFS, S3FS, "delta-spark~=1.0"],
+    "spark.DeltaTableDataSet": [SPARK, HDFS, S3FS, "delta-spark>=1.0, <3.0"],
 }
 svmlight_require = {"svmlight.SVMLightDataSet": ["scikit-learn~=1.0.2", "scipy~=1.7.3"]}
 tensorflow_require = {