Allow more image types for a spatial_experiment. (#38)

This introduces an IMAGE interface and an OTHER image_format that allows the
spatial_experiment to be extended to store more image types.

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.24)
 
 project(takane
-    VERSION 0.6.2
+    VERSION 0.7.0
     DESCRIPTION "ArtifactDB file validators"
     LANGUAGES CXX)
 

README.md

@@ -70,7 +70,8 @@ Currently, **takane** provides validators for the following objects:
 - `single_cell_experiment`:
   [1.0](https://github.com/ArtifactDB/takane/tree/gh-pages/docs/specifications/single_cell_experiment/1.0.md).
 - `spatial_experiment`:
-  [1.0](https://github.com/ArtifactDB/takane/tree/gh-pages/docs/specifications/spatial_experiment/1.0.md).
+  [1.0](https://github.com/ArtifactDB/takane/tree/gh-pages/docs/specifications/spatial_experiment/1.0.md),
+  [1.1](https://github.com/ArtifactDB/takane/tree/gh-pages/docs/specifications/spatial_experiment/1.1.md).
 - `string_factor`:
   [1.0](https://github.com/ArtifactDB/takane/tree/gh-pages/docs/specifications/string_factor/1.0.md).
 - `summarized_experiment`:

docs/specifications/_build.R

@@ -3,7 +3,8 @@ library(knitr)
 listings <- list.files(pattern="\\.Rmd$")
 default <- "1.0"
 known.variants <- list(
-    simple_list.Rmd="1.1"
+    simple_list.Rmd="1.1",
+    spatial_experiment.Rmd="1.1"
 )
 
 dest <- "compiled"

docs/specifications/_general.md

@@ -78,6 +78,7 @@ Known interfaces are listed below:
   Any number of assays are allowed.
   Any object may be used as an assay, as long as it has at least 2 dimensions with extents equal to that of the object itself.
   The row and column data should satisfy the `DATA_FRAME` interface, and should have number of rows equal to the number of rows or columns, respectively, of the object itself.
+- `IMAGE`: an image of arbitrary format.
 
 Different object types that satisfy the same object interface may not have the same on-disk representation.
 For example, a `vcf_experiment` satisfies the `SUMMARIZED_EXPERIMENT` interface but has a completely different on-disk representation from a `summarized_experiment`. 

docs/specifications/spatial_experiment.Rmd

@@ -58,21 +58,41 @@ This file should contain a `spatial_experiment` group, which contains the follow
 - `image_formats`, a 1-dimensional string dataset specifying the format for each image.
   This should have the same length as `image_samples`.
   The datatype should be representable by a UTF-8 encoded string.
-  Values should be either `"PNG"` or `"TIFF"`.
+```{r, results="asis", echo=FALSE}
+if (.version >= package_version("1.1")) {
+    cat('  Values should be either `"PNG"`, `"TIFF"` or `"OTHER"`.')
+} else {
+    cat('  Values should be either `"PNG"` or `"TIFF"`.')
+}
+```
 - `image_scale_factor`, a 1-dimensional dataset containing the scaling factor for each image.
   This should have the same length as `image_samples`.
   The datatype of this dataset is representable by a 64-bit float.
   Each value should be positive; multiplying the coordinates in `coordinates` by the scaling factor converts them into pixel coordinates in the corresponding image.
 
 The number of images is determined from the length of `image_samples` in the `images/mapping.h5` file.
-Each image should be stored in an image file inside the `images` subdirectory, named after its positional index, e.g., `0` for the first image, `1` for the second image.
+```{r, results="asis", echo=FALSE}
+if (.version >= package_version("1.1")) {
+    cat("Each image should be represented by a file or directory inside the `images` subdirectory, named after its positional index, e.g., `0` for the first image, `1` for the second image.
 The exact name is determined from `image_formats`:
 
-- For PNG files, a `.png` file extension should be added.
-- For PNG files, a `.tif` file extension should be added.
+- For PNG images, a `.png` file extension should be added.
+  Thus, the full name of the file would look like `0.png` for the first image, `1.png` for the second image, and so on.
+- For TIFF images, a `.tif` file extension should be added.
+  Thus, the full name of the file would look like `0.tif` for the first image, `1.tif` for the second image, and so on.
+- For OTHER images, the positional index is directly used as the directory name inside `images`.
+  This directory should contain a child object that satisfies the `IMAGE` interface.")
+} else {
+    cat("Each image should be represented by a file inside the `images` subdirectory, named after its positional index, e.g., `0` for the first image, `1` for the second image.
+The exact name is determined from `image_formats`:
+
+- For PNG images, a `.png` file extension should be added.
+- For TIFF images, a `.tif` file extension should be added.
 
 Thus, the full name of the files would look like `0.png` for the first image, `1.tif` for the second image, and so on depending on the format.
-Each image file should start with the magic numbers for the expected format.
+Each image file should start with the magic numbers for its expected format.")
+}
+```
 
 ## Height
 

include/takane/spatial_experiment.hpp

@@ -27,6 +27,8 @@ namespace takane {
  * @cond
  */
 bool derived_from(const std::string&, const std::string&, const Options& options);
+void validate(const std::filesystem::path&, const ObjectMetadata&, Options& options);
+bool satisfies_interface(const std::string&, const std::string&, const Options& options);
 /**
  * @endcond
  */
@@ -76,7 +78,7 @@ inline void validate_coordinates(const std::filesystem::path& path, size_t ncols
     }
 }
 
-inline void validate_image(const std::filesystem::path& path, size_t i, const std::string& format) {
+inline void validate_image(const std::filesystem::path& path, size_t i, const std::string& format, Options& options, const ritsuko::Version& version) {
     auto ipath = path / std::to_string(i);
 
     if (format == "PNG") {
@@ -96,12 +98,19 @@ inline void validate_image(const std::filesystem::path& path, size_t i, const st
             throw std::runtime_error("incorrect TIFF file signature for '" + ipath.string() + "'");
         }
 
+    } else if (format == "OTHER" && version.ge(1, 1, 0)) {
+        auto imeta = read_object_metadata(ipath);
+        if (!satisfies_interface(imeta.type, "IMAGE", options)) {
+            throw std::runtime_error("object in '" + ipath.string() + "' should satisfy the 'IMAGE' interface");
+        }
+        ::takane::validate(ipath, imeta, options);
+
     } else {
         throw std::runtime_error("image format '" + format + "' is not currently supported");
     }
 }
 
-inline void validate_images(const std::filesystem::path& path, size_t ncols, Options& options) {
+inline void validate_images(const std::filesystem::path& path, size_t ncols, Options& options, const ritsuko::Version& version) {
     auto image_dir = path / "images";
     auto mappath = image_dir / "mapping.h5";
     auto ihandle = ritsuko::hdf5::open_file(mappath);
@@ -198,7 +207,7 @@ inline void validate_images(const std::filesystem::path& path, size_t ncols, Opt
     // Now validating the images themselves.
     size_t num_images = image_formats.size();
     for (size_t i = 0; i < num_images; ++i) {
-        validate_image(image_dir, i, image_formats[i]);
+        validate_image(image_dir, i, image_formats[i], options, version);
     }
 
     size_t num_dir_obj = internal_other::count_directory_entries(image_dir);
@@ -228,7 +237,7 @@ inline void validate(const std::filesystem::path& path, const ObjectMetadata& me
 
     auto dims = ::takane::summarized_experiment::dimensions(path, metadata, options);
     internal::validate_coordinates(path, dims[1], options);
-    internal::validate_images(path, dims[1], options);
+    internal::validate_images(path, dims[1], options, version);
 }
 
 }

tests/src/spatial_experiment.cpp

@@ -284,6 +284,69 @@ TEST_F(SpatialExperimentTest, ImageFormats) {
     expect_error("not currently supported");
 }
 
+TEST_F(SpatialExperimentTest, OtherImageFormats) {
+    spatial_experiment::Options options(20, 19);
+    options.num_samples = 2;
+    options.num_images_per_sample = 7;
+    spatial_experiment::mock(dir, options);
+
+    // Bumping the version so we actually get support for OTHER image types.
+    {
+        auto opath = dir / "OBJECT";
+        auto parsed = millijson::parse_file(opath.c_str());
+        auto& remap = reinterpret_cast<millijson::Object*>(parsed.get())->values;
+        remap["type"] = std::shared_ptr<millijson::Base>(new millijson::String("spatial_experiment"));
+        spatial_experiment::add_object_metadata(parsed.get(), "1.1");
+        json_utils::dump(parsed.get(), opath);
+    }
+
+    size_t num_images = options.num_samples * options.num_images_per_sample;
+    {
+        H5::H5File handle(dir / "images" / "mapping.h5", H5F_ACC_RDWR);
+        auto ghandle = handle.openGroup("spatial_experiment");
+        ghandle.unlink("image_formats");
+        std::vector<std::string> replacement(num_images, "OTHER");
+        hdf5_utils::spawn_string_data(ghandle, "image_formats", H5T_VARIABLE, replacement);
+    }
+
+    {
+        auto idir = dir / "images";
+        for (const auto & entry : std::filesystem::directory_iterator(idir)) {
+            if (entry.path().filename() != "mapping.h5") {
+                std::filesystem::remove(entry.path());
+            }
+        }
+
+        for (size_t i = 0; i < num_images; ++i) {
+            auto ipath = idir / std::to_string(i);
+            std::filesystem::remove(ipath);
+            std::filesystem::create_directory(ipath);
+
+            auto optr = new millijson::Object;
+            std::shared_ptr<millijson::Base> contents(optr);
+            optr->values["type"] = std::shared_ptr<millijson::Base>(new millijson::String("some_image_class"));
+            optr->values["version"] = std::shared_ptr<millijson::Base>(new millijson::String("1.0"));
+            json_utils::dump(contents.get(), ipath / "OBJECT");
+        }
+    }
+    expect_error("satisfy the 'IMAGE' interface");
+
+    takane::Options vopt;
+    vopt.custom_satisfies_interface["IMAGE"].insert("some_image_class");
+    EXPECT_ANY_THROW({
+        try {
+            test_validate(dir, vopt);
+        } catch (std::exception& e) {
+            EXPECT_THAT(e.what(), ::testing::HasSubstr("no registered"));
+            throw;
+        }
+    });
+
+    // Mocking up a no-op function for our new class.
+    vopt.custom_validate["some_image_class"] = [](const std::filesystem::path&, const takane::ObjectMetadata&, takane::Options&) -> void {};
+    test_validate(dir, vopt);
+}
+
 TEST_F(SpatialExperimentTest, ImageSignature) {
     spatial_experiment::Options options(20, 19);
     options.num_samples = 1;