From e856c2cc1bd51be8f40bc1da99cc2b1e579fd2a8 Mon Sep 17 00:00:00 2001 From: Nicolas Peschke Date: Wed, 15 Sep 2021 19:43:09 +0200 Subject: [PATCH] fixing loading from txt file with dask (#24) * fixing loading from txt file hotfix for https://github.com/brainglobe/cellfinder/issues/184 * added unittests for IO * fixed side effect with tests searching for subdirectories moved brain_paths.txt to brain directory replaced paths with constants --- cellfinder_core/tools/IO.py | 7 +- tests/data/brain/brain_paths.txt | 270 ++++++++++++++++++++ tests/tests/test_unit/test_tools/test_IO.py | 18 ++ 3 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 tests/data/brain/brain_paths.txt create mode 100644 tests/tests/test_unit/test_tools/test_IO.py diff --git a/cellfinder_core/tools/IO.py b/cellfinder_core/tools/IO.py index b5baea0c..8b1c5c48 100644 --- a/cellfinder_core/tools/IO.py +++ b/cellfinder_core/tools/IO.py @@ -33,7 +33,12 @@ def read_with_dask(path): :return: """ - filenames = glob.glob(os.path.join(path, "*.tif")) + if path.endswith(".txt"): + with open(path, "r") as f: + filenames = [line.rstrip() for line in f.readlines()] + + else: + filenames = glob.glob(os.path.join(path, "*.tif")) shape, dtype = get_tiff_meta(filenames[0]) lazy_arrays = [lazy_imread(fn) for fn in get_sorted_file_paths(filenames)] diff --git a/tests/data/brain/brain_paths.txt b/tests/data/brain/brain_paths.txt new file mode 100644 index 00000000..936dc9bd --- /dev/null +++ b/tests/data/brain/brain_paths.txt @@ -0,0 +1,270 @@ +tests/data/brain/image_0000.tif +tests/data/brain/image_0001.tif +tests/data/brain/image_0002.tif +tests/data/brain/image_0003.tif +tests/data/brain/image_0004.tif +tests/data/brain/image_0005.tif +tests/data/brain/image_0006.tif +tests/data/brain/image_0007.tif +tests/data/brain/image_0008.tif +tests/data/brain/image_0009.tif +tests/data/brain/image_0010.tif +tests/data/brain/image_0011.tif +tests/data/brain/image_0012.tif +tests/data/brain/image_0013.tif +tests/data/brain/image_0014.tif +tests/data/brain/image_0015.tif +tests/data/brain/image_0016.tif +tests/data/brain/image_0017.tif +tests/data/brain/image_0018.tif +tests/data/brain/image_0019.tif +tests/data/brain/image_0020.tif +tests/data/brain/image_0021.tif +tests/data/brain/image_0022.tif +tests/data/brain/image_0023.tif +tests/data/brain/image_0024.tif +tests/data/brain/image_0025.tif +tests/data/brain/image_0026.tif +tests/data/brain/image_0027.tif +tests/data/brain/image_0028.tif +tests/data/brain/image_0029.tif +tests/data/brain/image_0030.tif +tests/data/brain/image_0031.tif +tests/data/brain/image_0032.tif +tests/data/brain/image_0033.tif +tests/data/brain/image_0034.tif +tests/data/brain/image_0035.tif +tests/data/brain/image_0036.tif +tests/data/brain/image_0037.tif +tests/data/brain/image_0038.tif +tests/data/brain/image_0039.tif +tests/data/brain/image_0040.tif +tests/data/brain/image_0041.tif +tests/data/brain/image_0042.tif +tests/data/brain/image_0043.tif +tests/data/brain/image_0044.tif +tests/data/brain/image_0045.tif +tests/data/brain/image_0046.tif +tests/data/brain/image_0047.tif +tests/data/brain/image_0048.tif +tests/data/brain/image_0049.tif +tests/data/brain/image_0050.tif +tests/data/brain/image_0051.tif +tests/data/brain/image_0052.tif +tests/data/brain/image_0053.tif +tests/data/brain/image_0054.tif +tests/data/brain/image_0055.tif +tests/data/brain/image_0056.tif +tests/data/brain/image_0057.tif +tests/data/brain/image_0058.tif +tests/data/brain/image_0059.tif +tests/data/brain/image_0060.tif +tests/data/brain/image_0061.tif +tests/data/brain/image_0062.tif +tests/data/brain/image_0063.tif +tests/data/brain/image_0064.tif +tests/data/brain/image_0065.tif +tests/data/brain/image_0066.tif +tests/data/brain/image_0067.tif +tests/data/brain/image_0068.tif +tests/data/brain/image_0069.tif +tests/data/brain/image_0070.tif +tests/data/brain/image_0071.tif +tests/data/brain/image_0072.tif +tests/data/brain/image_0073.tif +tests/data/brain/image_0074.tif +tests/data/brain/image_0075.tif +tests/data/brain/image_0076.tif +tests/data/brain/image_0077.tif +tests/data/brain/image_0078.tif +tests/data/brain/image_0079.tif +tests/data/brain/image_0080.tif +tests/data/brain/image_0081.tif +tests/data/brain/image_0082.tif +tests/data/brain/image_0083.tif +tests/data/brain/image_0084.tif +tests/data/brain/image_0085.tif +tests/data/brain/image_0086.tif +tests/data/brain/image_0087.tif +tests/data/brain/image_0088.tif +tests/data/brain/image_0089.tif +tests/data/brain/image_0090.tif +tests/data/brain/image_0091.tif +tests/data/brain/image_0092.tif +tests/data/brain/image_0093.tif +tests/data/brain/image_0094.tif +tests/data/brain/image_0095.tif +tests/data/brain/image_0096.tif +tests/data/brain/image_0097.tif +tests/data/brain/image_0098.tif +tests/data/brain/image_0099.tif +tests/data/brain/image_0100.tif +tests/data/brain/image_0101.tif +tests/data/brain/image_0102.tif +tests/data/brain/image_0103.tif +tests/data/brain/image_0104.tif +tests/data/brain/image_0105.tif +tests/data/brain/image_0106.tif +tests/data/brain/image_0107.tif +tests/data/brain/image_0108.tif +tests/data/brain/image_0109.tif +tests/data/brain/image_0110.tif +tests/data/brain/image_0111.tif +tests/data/brain/image_0112.tif +tests/data/brain/image_0113.tif +tests/data/brain/image_0114.tif +tests/data/brain/image_0115.tif +tests/data/brain/image_0116.tif +tests/data/brain/image_0117.tif +tests/data/brain/image_0118.tif +tests/data/brain/image_0119.tif +tests/data/brain/image_0120.tif +tests/data/brain/image_0121.tif +tests/data/brain/image_0122.tif +tests/data/brain/image_0123.tif +tests/data/brain/image_0124.tif +tests/data/brain/image_0125.tif +tests/data/brain/image_0126.tif +tests/data/brain/image_0127.tif +tests/data/brain/image_0128.tif +tests/data/brain/image_0129.tif +tests/data/brain/image_0130.tif +tests/data/brain/image_0131.tif +tests/data/brain/image_0132.tif +tests/data/brain/image_0133.tif +tests/data/brain/image_0134.tif +tests/data/brain/image_0135.tif +tests/data/brain/image_0136.tif +tests/data/brain/image_0137.tif +tests/data/brain/image_0138.tif +tests/data/brain/image_0139.tif +tests/data/brain/image_0140.tif +tests/data/brain/image_0141.tif +tests/data/brain/image_0142.tif +tests/data/brain/image_0143.tif +tests/data/brain/image_0144.tif +tests/data/brain/image_0145.tif +tests/data/brain/image_0146.tif +tests/data/brain/image_0147.tif +tests/data/brain/image_0148.tif +tests/data/brain/image_0149.tif +tests/data/brain/image_0150.tif +tests/data/brain/image_0151.tif +tests/data/brain/image_0152.tif +tests/data/brain/image_0153.tif +tests/data/brain/image_0154.tif +tests/data/brain/image_0155.tif +tests/data/brain/image_0156.tif +tests/data/brain/image_0157.tif +tests/data/brain/image_0158.tif +tests/data/brain/image_0159.tif +tests/data/brain/image_0160.tif +tests/data/brain/image_0161.tif +tests/data/brain/image_0162.tif +tests/data/brain/image_0163.tif +tests/data/brain/image_0164.tif +tests/data/brain/image_0165.tif +tests/data/brain/image_0166.tif +tests/data/brain/image_0167.tif +tests/data/brain/image_0168.tif +tests/data/brain/image_0169.tif +tests/data/brain/image_0170.tif +tests/data/brain/image_0171.tif +tests/data/brain/image_0172.tif +tests/data/brain/image_0173.tif +tests/data/brain/image_0174.tif +tests/data/brain/image_0175.tif +tests/data/brain/image_0176.tif +tests/data/brain/image_0177.tif +tests/data/brain/image_0178.tif +tests/data/brain/image_0179.tif +tests/data/brain/image_0180.tif +tests/data/brain/image_0181.tif +tests/data/brain/image_0182.tif +tests/data/brain/image_0183.tif +tests/data/brain/image_0184.tif +tests/data/brain/image_0185.tif +tests/data/brain/image_0186.tif +tests/data/brain/image_0187.tif +tests/data/brain/image_0188.tif +tests/data/brain/image_0189.tif +tests/data/brain/image_0190.tif +tests/data/brain/image_0191.tif +tests/data/brain/image_0192.tif +tests/data/brain/image_0193.tif +tests/data/brain/image_0194.tif +tests/data/brain/image_0195.tif +tests/data/brain/image_0196.tif +tests/data/brain/image_0197.tif +tests/data/brain/image_0198.tif +tests/data/brain/image_0199.tif +tests/data/brain/image_0200.tif +tests/data/brain/image_0201.tif +tests/data/brain/image_0202.tif +tests/data/brain/image_0203.tif +tests/data/brain/image_0204.tif +tests/data/brain/image_0205.tif +tests/data/brain/image_0206.tif +tests/data/brain/image_0207.tif +tests/data/brain/image_0208.tif +tests/data/brain/image_0209.tif +tests/data/brain/image_0210.tif +tests/data/brain/image_0211.tif +tests/data/brain/image_0212.tif +tests/data/brain/image_0213.tif +tests/data/brain/image_0214.tif +tests/data/brain/image_0215.tif +tests/data/brain/image_0216.tif +tests/data/brain/image_0217.tif +tests/data/brain/image_0218.tif +tests/data/brain/image_0219.tif +tests/data/brain/image_0220.tif +tests/data/brain/image_0221.tif +tests/data/brain/image_0222.tif +tests/data/brain/image_0223.tif +tests/data/brain/image_0224.tif +tests/data/brain/image_0225.tif +tests/data/brain/image_0226.tif +tests/data/brain/image_0227.tif +tests/data/brain/image_0228.tif +tests/data/brain/image_0229.tif +tests/data/brain/image_0230.tif +tests/data/brain/image_0231.tif +tests/data/brain/image_0232.tif +tests/data/brain/image_0233.tif +tests/data/brain/image_0234.tif +tests/data/brain/image_0235.tif +tests/data/brain/image_0236.tif +tests/data/brain/image_0237.tif +tests/data/brain/image_0238.tif +tests/data/brain/image_0239.tif +tests/data/brain/image_0240.tif +tests/data/brain/image_0241.tif +tests/data/brain/image_0242.tif +tests/data/brain/image_0243.tif +tests/data/brain/image_0244.tif +tests/data/brain/image_0245.tif +tests/data/brain/image_0246.tif +tests/data/brain/image_0247.tif +tests/data/brain/image_0248.tif +tests/data/brain/image_0249.tif +tests/data/brain/image_0250.tif +tests/data/brain/image_0251.tif +tests/data/brain/image_0252.tif +tests/data/brain/image_0253.tif +tests/data/brain/image_0254.tif +tests/data/brain/image_0255.tif +tests/data/brain/image_0256.tif +tests/data/brain/image_0257.tif +tests/data/brain/image_0258.tif +tests/data/brain/image_0259.tif +tests/data/brain/image_0260.tif +tests/data/brain/image_0261.tif +tests/data/brain/image_0262.tif +tests/data/brain/image_0263.tif +tests/data/brain/image_0264.tif +tests/data/brain/image_0265.tif +tests/data/brain/image_0266.tif +tests/data/brain/image_0267.tif +tests/data/brain/image_0268.tif +tests/data/brain/image_0269.tif diff --git a/tests/tests/test_unit/test_tools/test_IO.py b/tests/tests/test_unit/test_tools/test_IO.py new file mode 100644 index 00000000..a649a577 --- /dev/null +++ b/tests/tests/test_unit/test_tools/test_IO.py @@ -0,0 +1,18 @@ +import dask.array as d_array + +from cellfinder_core.tools import IO + +BRAIN_DIR = "tests/data/brain" +BRAIN_PATHS = f"{BRAIN_DIR}/brain_paths.txt" + + +def test_read_with_dask_txt(): + stack = IO.read_with_dask(BRAIN_PATHS) + assert type(stack) == d_array.Array + + +def test_read_with_dask_glob_txt_equal(): + txt_stack = IO.read_with_dask(BRAIN_PATHS) + glob_stack = IO.read_with_dask(BRAIN_DIR) + + assert d_array.equal(txt_stack, glob_stack).all()