Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PERF: delay frontends initialization until yt.load is actually called #4539

Merged
merged 3 commits into from
Jul 3, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions yt/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
derived_field,
field_plugins,
)
from yt.frontends.api import _frontend_container
from yt.funcs import (
enable_plugins,
get_memory_usage,
Expand Down Expand Up @@ -71,7 +70,7 @@
from yt.units.unit_object import define_unit # type: ignore
from yt.utilities.logger import set_log_level, ytLogger as mylog

frontends = _frontend_container()
from yt import frontends

import yt.visualization.volume_rendering.api as volume_rendering
from yt.frontends.stream.api import hexahedral_connectivity
Expand Down
2 changes: 2 additions & 0 deletions yt/data_objects/tests/test_registration.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@


def test_reregistration_warning():
from yt.frontends import enzo # noqa

true_EnzoDataset = output_type_registry["EnzoDataset"]
try:
with pytest.warns(
Expand Down
58 changes: 58 additions & 0 deletions yt/frontends/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
__all__ = [
"adaptahop",
"ahf",
"amrvac",
"art",
"arepo",
"artio",
"athena",
"athena_pp",
"boxlib",
"cf_radial",
"chimera",
"chombo",
"cholla",
"eagle",
"enzo_e",
"enzo",
"exodus_ii",
"fits",
"flash",
"gadget",
"gadget_fof",
"gamer",
"gdf",
"gizmo",
"halo_catalog",
"http_stream",
"moab",
"nc4_cm1",
"open_pmd",
"owls",
"owls_subfind",
"ramses",
"rockstar",
"sdf",
"stream",
"swift",
"tipsy",
"ytdata",
]

from functools import lru_cache


@lru_cache(maxsize=None)
def __getattr__(value):
import importlib

if value == "_all":
neutrinoceros marked this conversation as resolved.
Show resolved Hide resolved
# recursively import all frontends
for _ in __all__:
__getattr__(_)
return

if value not in __all__:
raise AttributeError(f"yt.frontends has no attribute {value!r}")

return importlib.import_module(f"yt.frontends.{value}.api")
57 changes: 1 addition & 56 deletions yt/frontends/api.py
Original file line number Diff line number Diff line change
@@ -1,56 +1 @@
import glob
import importlib
import os
import sys
import time
import types

_frontends = [
"adaptahop",
"ahf",
"amrvac",
"art",
"arepo",
"artio",
"athena",
"athena_pp",
"boxlib",
"cf_radial",
"chimera",
"chombo",
"cholla",
"eagle",
"enzo_e",
"enzo",
"exodus_ii",
"fits",
"flash",
"gadget",
"gadget_fof",
"gamer",
"gdf",
"gizmo",
"halo_catalog",
"http_stream",
"moab",
"nc4_cm1",
"open_pmd",
"owls",
"owls_subfind",
"ramses",
"rockstar",
"sdf",
"stream",
"swift",
"tipsy",
"ytdata",
]


class _frontend_container:
def __init__(self):
for frontend in _frontends:
_mod = f"yt.frontends.{frontend}.api"
setattr(self, frontend, importlib.import_module(_mod))
setattr(self, "api", importlib.import_module("yt.frontends.api"))
setattr(self, "__name__", "yt.frontends.api")
from . import __all__ as _frontends # backward compat
3 changes: 3 additions & 0 deletions yt/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ def load(
yt.utilities.exceptions.YTAmbiguousDataType
If the data format matches more than one class of similar specialization levels.
"""
from yt.frontends import _all # type: ignore [attr-defined] # noqa
Copy link
Contributor

@chrishavlin chrishavlin Jul 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh and one extra comment/question -- by nesting this import in the load and load_simulation calls, does that mean one could import yt and then directly instantiate a frontend without ever importing all the other frontends? That would actually be a pretty nice consequence beyond simply delaying the imports... might be worth a mention in the docs even. Could make a difference for batch processing scripts.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

indeed, I think with this patch you'd be able to short-circuit all frontends being loaded, but I think at this point it should be seen as an implementation detail rather than a feature (in particular because circumventing yt.load would require different code if your frontends isn't builtin, but lives in an extension)


fn = os.path.expanduser(fn)

if any(wildcard in fn for wildcard in "[]?!*"):
Expand Down Expand Up @@ -155,6 +157,7 @@ def load_simulation(fn, simulation_type, find_outputs=False):
yt.utilities.exceptions.YTSimulationNotIdentified
If simulation_type is unknown.
"""
from yt.frontends import _all # noqa

fn = str(lookup_on_disk_data(fn))

Expand Down