modify radx pipeline

helxplatform · Dec 18, 2024 · b41b4b3 · b41b4b3
1 parent dec80b9
commit b41b4b3
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/dags/roger/pipelines/radx.py b/dags/roger/pipelines/radx.py
@@ -1,8 +1,18 @@
 "Pipeline for BACPAC data"
 
 from roger.pipelines import DugPipeline
+from roger.core import storage
+
 
 class RadxPipeline(DugPipeline):
-    "Pipeline for BACPAC data set"
+    "Pipeline for Radx data set"
     pipeline_name = "radx"
     parser_name = "radx"
+
+    def get_objects(self, input_data_path=None):
+        if not input_data_path:
+            input_data_path = storage.dug_kfdrc_path()
+        files = storage.get_files_recursive(
+            lambda file_name: file_name.endswith('.json'),
+            input_data_path)
+        return sorted([str(f) for f in files])
diff --git a/requirements.txt b/requirements.txt
@@ -7,7 +7,7 @@ redisgraph-bulk-loader==0.12.3
 setuptools>=66
 pytest
 PyYAML
-git+https://github.com/helxplatform/dug@2.13.4
+git+https://github.com/helxplatform/dug@radx-v1
 orjson
 kg-utils==0.0.6
 bmt==1.1.0