~More completion of v15 ETL, cleanup

ImagingDataCommons · Jun 26, 2023 · 702b120 · 702b120
1 parent a2368cc
commit 702b120
Show file tree

Hide file tree

Showing 63 changed files with 52 additions and 3,800 deletions.
diff --git a/bq/utils/publish_dataset/publish_idc_pdp_staging_datasets.py b/bq/utils/publish_dataset/publish_idc_pdp_staging_datasets.py
@@ -33,7 +33,7 @@
     progresslogger.info(f'args: {json.dumps(args.__dict__, indent=2)}')
 
     for src_dataset in (
-            'idc_v14',
-            'idc_v14_clinical',
+            f'idc_v{settings.CURRENT_VERSION}',
+            f'idc_v1{settings.CURRENT_VERSION}_clinical',
             ):
         publish_dataset(args)
diff --git a/gcs/copy_blobs_using_BQ_query/copy_apollo_blobs.py b/gcs/copy_blobs_using_BQ_query/copy_apollo_blobs.py
diff --git a/gcs/copy_blobs_using_BQ_query/copy_pathology_blobs.py b/gcs/copy_blobs_using_BQ_query/copy_pathology_blobs.py
diff --git a/gcs/copy_blobs_using_BQ_query/pathology_collections.py b/gcs/copy_blobs_using_BQ_query/pathology_collections.py
diff --git a/gcs/copy_bucket_mp/copy_bucket_mp.py → gcs/copy_bucket_mp.py b/gcs/copy_bucket_mp/copy_bucket_mp.py → gcs/copy_bucket_mp.py
@@ -90,8 +90,6 @@ def worker(input, args, dones):
 def copy_all_instances(args, dones):
     client = storage.Client()
     src_bucket = storage.Bucket(client, args.src_bucket)
-
-
     n=len(dones)
 
     progresslogger.info(f"{len(dones)} blobs previously copied")

diff --git a/...o_dev_buckets/copy_premerge_to_staging.py → ...ge_to_dev_buckets/copy_premerge_to_dev.py b/...o_dev_buckets/copy_premerge_to_staging.py → ...ge_to_dev_buckets/copy_premerge_to_dev.py
diff --git a/gcs/misc/README.md b/gcs/misc/README.md
diff --git a/gcs/misc/delete_list_of_blobs/delete_idc_dev_defaced_pathology.py b/gcs/misc/delete_list_of_blobs/delete_idc_dev_defaced_pathology.py
diff --git a/gcs/misc/delete_list_of_blobs/delete_list_of_blobs.py b/gcs/misc/delete_list_of_blobs/delete_list_of_blobs.py