FileNotFoundError - InMemoryFITSFile error related to FITSDataset

[yzhenggit]

Bug report

Bug summary

Hello! I'm trying to convert a 3D radio data (velocity, position, position) to a data source in yt, but kept getting the FileNotFoundError (InMemoryFITSFile xxx). See below. Thank you!

Code for reproduction

cubefile = 'example_cube.fits' # download data at  http://use.yt/upload/22205891

import astropy.io.fits as fits
cube = fits.getdata(cubefile)
header = fits.getheader(cubefile)

from astropy.io.fits import PrimaryHDU
hdu = PrimaryHDU(cube, header=header)

from yt.frontends.fits.api import SpectralCubeFITSDataset as FITSDataset
ds = FITSDataset(hdu)

Actual outcome

yt : [WARNING  ] 2023-06-21 19:16:25,710 Cannot find time
yt : [INFO     ] 2023-06-21 19:16:25,712 Detected these axes: RA---ARC DEC--ARC VRAD 
yt : [WARNING  ] 2023-06-21 19:16:25,715 No length conversion provided. Assuming 1 = 1 cm.
yt : [WARNING  ] 2023-06-21 19:16:25,716 Assuming 1.0 = 1.0 s
yt : [WARNING  ] 2023-06-21 19:16:25,716 Assuming 1.0 = 1.0 g
yt : [INFO     ] 2023-06-21 19:16:25,752 Parameters: current_time              = 0.0
yt : [INFO     ] 2023-06-21 19:16:25,753 Parameters: domain_dimensions         = [3 4 7]
yt : [INFO     ] 2023-06-21 19:16:25,754 Parameters: domain_left_edge          = [0.5 0.5 0.5]
yt : [INFO     ] 2023-06-21 19:16:25,755 Parameters: domain_right_edge         = [3.5 4.5 7.5]
yt : [INFO     ] 2023-06-21 19:16:25,755 Parameters: cosmological_simulation   = 0
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[1], line 13
     10 hdu = PrimaryHDU(cube, header=header)
     12 from yt.frontends.fits.api import SpectralCubeFITSDataset as FITSDataset
---> 13 ds = FITSDataset(hdu)

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/data_objects/static_output.py:220, in Dataset.__new__(cls, filename, *args, **kwargs)
    216     is_stream = hasattr(filename, "get_fields") and hasattr(
    217         filename, "get_particle_type"
    218     )
    219     if not is_stream:
--> 220         obj.__init__(filename, *args, **kwargs)
    221     return obj
    222 apath = os.path.abspath(os.path.expanduser(filename))

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/frontends/fits/data_structures.py:760, in SpectralCubeFITSDataset.__init__(self, filename, auxiliary_files, nprocs, storage_filename, nan_mask, spectral_factor, suppress_astropy_warnings, parameters, units_override, unit_system)
    758     auxiliary_files = []
    759 self.spectral_factor = spectral_factor
--> 760 super().__init__(
    761     filename,
    762     nprocs=nprocs,
    763     auxiliary_files=auxiliary_files,
    764     storage_filename=storage_filename,
    765     suppress_astropy_warnings=suppress_astropy_warnings,
    766     nan_mask=nan_mask,
    767     parameters=parameters,
    768     units_override=units_override,
    769     unit_system=unit_system,
    770 )

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/frontends/fits/data_structures.py:379, in FITSDataset.__init__(self, filename, dataset_type, auxiliary_files, nprocs, storage_filename, nan_mask, suppress_astropy_warnings, parameters, units_override, unit_system)
    375         self._handle._fits_files.append(f)
    377 self.refine_by = 2
--> 379 Dataset.__init__(
    380     self,
    381     fn,
    382     dataset_type,
    383     units_override=units_override,
    384     unit_system=unit_system,
    385 )
    386 self.storage_filename = storage_filename

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/data_objects/static_output.py:321, in Dataset.__init__(self, filename, dataset_type, units_override, unit_system, default_species_fields, axis_order)
    316     raise RuntimeError(
    317         "Something went wrong during yt's initialization: "
    318         "dataset cache isn't properly initialized"
    319     )
    320 try:
--> 321     _ds_store.check_ds(self)
    322 except NoParameterShelf:
    323     pass

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/utilities/parameter_file_storage.py:135, in ParameterFileStore.check_ds(self, ds)
    129 def check_ds(self, ds):
    130     """
    131     This will ensure that the dataset (*ds*) handed to it is
    132     recorded in the storage unit.  In doing so, it will update path
    133     and "last_seen" information.
    134     """
--> 135     hash = ds._hash()
    136     if hash not in self._records:
    137         self.insert_ds(ds)

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/data_objects/static_output.py:439, in Dataset._hash(self)
    438 def _hash(self):
--> 439     s = f"{self.basename};{self.current_time};{self.unique_identifier}"
    440     return hashlib.md5(s.encode("utf-8")).hexdigest()

File ~/anaconda3/envs/env_3dcube/lib/python3.11/functools.py:1001, in cached_property.__get__(self, instance, owner)
    999 val = cache.get(self.attrname, _NOT_FOUND)
   1000 if val is _NOT_FOUND:
-> 1001     val = self.func(instance)
   1002     try:
   1003         cache[self.attrname] = val

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/frontends/fits/data_structures.py:434, in FITSDataset.unique_identifier(self)
    432     return str(time.time())
    433 else:
--> 434     return super().unique_identifier

File ~/anaconda3/envs/env_3dcube/lib/python3.11/functools.py:1001, in cached_property.__get__(self, instance, owner)
    999 val = cache.get(self.attrname, _NOT_FOUND)
   1000 if val is _NOT_FOUND:
-> 1001     val = self.func(instance)
   1002     try:
   1003         cache[self.attrname] = val

File ~/anaconda3/envs/env_3dcube/lib/python3.11/site-packages/yt/data_objects/static_output.py:363, in Dataset.unique_identifier(self)
    361 @cached_property
    362 def unique_identifier(self) -> str:
--> 363     retv = int(os.stat(self.parameter_filename)[ST_CTIME])
    364     name_as_bytes = bytearray(map(ord, self.parameter_filename))
    365     retv += fnv_hash(name_as_bytes)

FileNotFoundError: [Errno 2] No such file or directory: '/Users/Yong/Dropbox/Mac/Desktop/InMemoryFITSFile_674fb864e1fb4892bc52346402773692'

Expected outcome

I was expecting the FITSDataset(hdu) to successful produce a data object.

Version Information

• Operating System: macOS Monterey version 12.3.1
• Python Version: 3.11.4
• yt version: 4.2.1 (default conda installation)
• Other Libraries (if applicable):

[neutrinoceros]

Thank you for reporting and providing a minimal reproducer !
I can reproduce the error on the main branch, and down to yt 4.1.0, but the script appears to run fine on yt 4.0.5, so this is likely a regression. I'll bisect more precisely.

[neutrinoceros]

Bisects to #3772 (cd31314)

[yzhenggit]

Thank you for looking into this so swiftly!