From 2dec15a08f645fcfb0bc6be284771a82a6dc3a95 Mon Sep 17 00:00:00 2001 From: Britton Smith Date: Wed, 3 Mar 2021 11:03:27 +0000 Subject: [PATCH 01/60] Parallelize coarse index calculation. --- yt/geometry/particle_geometry_handler.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 0d5cdb75e24..59f03eb0d92 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -12,7 +12,8 @@ from yt.geometry.particle_oct_container import ParticleBitmap from yt.utilities.lib.fnv_hash import fnv_hash from yt.utilities.logger import ytLogger as mylog - +from yt.utilities.parallel_tools.parallel_analysis_interface import \ + parallel_objects class ParticleIndex(Index): """The Index subclass for particle datasets""" @@ -195,8 +196,8 @@ def _initialize_index(self): def _initialize_coarse_index(self): pb = get_pbar("Initializing coarse index ", len(self.data_files)) - for i, data_file in enumerate(self.data_files): - pb.update(i) + for i, data_file in parallel_objects(enumerate(self.data_files)): + pb.update(i+1) for ptype, pos in self.io._yield_coordinates(data_file): ds = self.ds if hasattr(ds, "_sph_ptypes") and ptype == ds._sph_ptypes[0]: @@ -206,8 +207,10 @@ def _initialize_coarse_index(self): else: hsml = None self.regions._coarse_index_data_file(pos, hsml, data_file.file_id) - self.regions._set_coarse_index_data_file(data_file.file_id) pb.finish() + self.regions.masks = self.comm.mpi_allreduce(self.regions.masks, op="sum") + for data_file in self.data_files: + self.regions._set_coarse_index_data_file(data_file.file_id) self.regions.find_collisions_coarse() def _initialize_refined_index(self): From 914f075420053cb4c681cff552b16e92316efecc Mon Sep 17 00:00:00 2001 From: Britton Smith Date: Wed, 3 Mar 2021 13:50:45 +0000 Subject: [PATCH 02/60] Parallelize refined index. --- yt/geometry/particle_geometry_handler.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 59f03eb0d92..e0e7c3c3e08 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -10,6 +10,7 @@ from yt.funcs import get_pbar, only_on_root from yt.geometry.geometry_handler import Index, YTDataChunk from yt.geometry.particle_oct_container import ParticleBitmap +from yt.utilities.lib.ewah_bool_wrap import BoolArrayCollection from yt.utilities.lib.fnv_hash import fnv_hash from yt.utilities.logger import ytLogger as mylog from yt.utilities.parallel_tools.parallel_analysis_interface import \ @@ -235,9 +236,11 @@ def _initialize_refined_index(self): total_coarse_refined, 100 * total_coarse_refined / mask.size, ) - for i, data_file in enumerate(self.data_files): + storage = {} + for sto, (i, data_file) in parallel_objects(enumerate(self.data_files), + storage=storage): coll = None - pb.update(i) + pb.update(i+1) nsub_mi = 0 for ptype, pos in self.io._yield_coordinates(data_file): if pos.size == 0: @@ -261,8 +264,13 @@ def _initialize_refined_index(self): mask_threshold=mask_threshold, ) total_refined += nsub_mi - self.regions.bitmasks.append(data_file.file_id, coll) + sto.result_id = i + sto.result = (data_file.file_id, coll.dumps()) pb.finish() + for i, (file_id, coll_str) in sorted(storage.items()): + coll = BoolArrayCollection() + coll.loads(coll_str) + self.regions.bitmasks.append(file_id, coll) self.regions.find_collisions_refined() def _detect_output_fields(self): From be1e53a03fbb021a667643e4218ee59a3d8ef327 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Mar 2021 16:57:13 +0000 Subject: [PATCH 03/60] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/geometry/particle_geometry_handler.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index e0e7c3c3e08..505edea8af8 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -13,8 +13,8 @@ from yt.utilities.lib.ewah_bool_wrap import BoolArrayCollection from yt.utilities.lib.fnv_hash import fnv_hash from yt.utilities.logger import ytLogger as mylog -from yt.utilities.parallel_tools.parallel_analysis_interface import \ - parallel_objects +from yt.utilities.parallel_tools.parallel_analysis_interface import parallel_objects + class ParticleIndex(Index): """The Index subclass for particle datasets""" @@ -198,7 +198,7 @@ def _initialize_index(self): def _initialize_coarse_index(self): pb = get_pbar("Initializing coarse index ", len(self.data_files)) for i, data_file in parallel_objects(enumerate(self.data_files)): - pb.update(i+1) + pb.update(i + 1) for ptype, pos in self.io._yield_coordinates(data_file): ds = self.ds if hasattr(ds, "_sph_ptypes") and ptype == ds._sph_ptypes[0]: @@ -237,10 +237,11 @@ def _initialize_refined_index(self): 100 * total_coarse_refined / mask.size, ) storage = {} - for sto, (i, data_file) in parallel_objects(enumerate(self.data_files), - storage=storage): + for sto, (i, data_file) in parallel_objects( + enumerate(self.data_files), storage=storage + ): coll = None - pb.update(i+1) + pb.update(i + 1) nsub_mi = 0 for ptype, pos in self.io._yield_coordinates(data_file): if pos.size == 0: From 2355900825a78deb40c5ca457c773025b865bf87 Mon Sep 17 00:00:00 2001 From: Britton Smith Date: Wed, 3 Mar 2021 17:05:56 +0000 Subject: [PATCH 04/60] Fix flake8 errors. --- yt/geometry/particle_geometry_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index e0e7c3c3e08..fb5d99a3951 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -198,7 +198,7 @@ def _initialize_index(self): def _initialize_coarse_index(self): pb = get_pbar("Initializing coarse index ", len(self.data_files)) for i, data_file in parallel_objects(enumerate(self.data_files)): - pb.update(i+1) + pb.update(i + 1) for ptype, pos in self.io._yield_coordinates(data_file): ds = self.ds if hasattr(ds, "_sph_ptypes") and ptype == ds._sph_ptypes[0]: @@ -240,7 +240,7 @@ def _initialize_refined_index(self): for sto, (i, data_file) in parallel_objects(enumerate(self.data_files), storage=storage): coll = None - pb.update(i+1) + pb.update(i + 1) nsub_mi = 0 for ptype, pos in self.io._yield_coordinates(data_file): if pos.size == 0: @@ -267,7 +267,8 @@ def _initialize_refined_index(self): sto.result_id = i sto.result = (data_file.file_id, coll.dumps()) pb.finish() - for i, (file_id, coll_str) in sorted(storage.items()): + for i in sorted(storage): + file_id, coll_str = storage[i] coll = BoolArrayCollection() coll.loads(coll_str) self.regions.bitmasks.append(file_id, coll) From 4c152fb3b26152eec6d78fb42ff9d0144fc6d5ff Mon Sep 17 00:00:00 2001 From: Britton Smith Date: Wed, 3 Mar 2021 20:24:16 +0000 Subject: [PATCH 05/60] Check if coll is None. --- yt/geometry/particle_geometry_handler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 551736fc501..8c5c8fc3346 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -266,7 +266,11 @@ def _initialize_refined_index(self): ) total_refined += nsub_mi sto.result_id = i - sto.result = (data_file.file_id, coll.dumps()) + if coll is None: + coll_str = b"" + else: + coll_str = coll.dumps() + sto.result = (data_file.file_id, coll_str) pb.finish() for i in sorted(storage): file_id, coll_str = storage[i] From b6216362b41e514217d96f0aed16290a9e0a6639 Mon Sep 17 00:00:00 2001 From: Jared Coughlin Date: Tue, 16 Mar 2021 17:22:59 -0500 Subject: [PATCH 06/60] Updates testing docs to reflect switch to pytest. --- doc/source/developing/testing.rst | 550 ++++++++++++------------------ 1 file changed, 221 insertions(+), 329 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 245e71e50c9..9584c999e85 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -3,143 +3,101 @@ Testing ======= -yt includes a testing suite which one can run on the codebase to assure that no -breaks in functionality have occurred. This testing suite is based on the Nose_ -testing framework. The suite consists of two components, unit tests and answer -tests. Unit tests confirm that an isolated piece of functionality runs without -failure for inputs with known correct outputs. Answer tests verify the -integration and compatibility of the individual code unit by generating output -from user-visible yt functions and comparing and matching the results against -outputs of the same function produced using older versions of the yt codebase. -This ensures consistency in results as development proceeds. - -.. _nosetests: +yt includes a testing suite that one can run on the code base to assure that no +breaks in functionality have occurred. This suite is based on the `pytest `_ testing framework and consists of two types of tests: + +* Unit tests. These confirm that an isolated piece of functionality runs without failure for inputs with known correct outputs. + +* Answer tests. These generate output from user-visible yt functions and compare this output against the output produced using an older, "known good", version of yt. + +These tests ensure consistency in results as development proceeds. The testing suite should be run locally by developers to make sure they aren't -checking in any code that breaks existing functionality. To further this goal, -an automatic buildbot runs the test suite after each code commit to confirm -that yt hasn't broken recently. To supplement this effort, we also maintain a -`continuous integration server `_ that runs the -tests with each commit to the yt version control repository. +checking in any code that breaks existing functionality. Additionally, as a safeguard, the test suite is automatically run after each commit that is pushed to the yt repository on github. .. _unit_testing: Unit Testing ------------ -What do Unit Tests Do +What Do Unit Tests Do ^^^^^^^^^^^^^^^^^^^^^ -Unit tests are tests that operate on some small set of machinery, and verify -that the machinery works. yt uses the `Nose -`_ framework for running unit tests. In -practice, what this means is that we write scripts that assert statements, and -Nose identifies those scripts, runs them, and verifies that the assertions are -true and the code runs without crashing. +Unit tests are tests that operate on some small piece of machinery and verify +that the machinery works. In +practice, this means that we make assertions about a piece of machinery and then pytest runs the machinery, verifies that the assertions are true, and ensures that the code runs without crashing. An example of a unit test is ``test_all_fields`` in ``yt/fields/tests/test_fields.py``. How to Run the Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^ -One can run the unit tests very straightforwardly from any python interpreter -that can import the yt module: - -.. code-block:: python - - import yt - - yt.run_nose() - -If you are developing new functionality, it is sometimes more convenient to use -the Nose command line interface, ``nosetests``. You can run the unit tests -using ``nose`` by navigating to the base directory of the yt git -repository and invoking ``nosetests``: +One can run the unit tests by navigating to the base directory of the yt git +repository and invoking ``pytest``: .. code-block:: bash $ cd $YT_GIT - $ nosetests + $ pytest where ``$YT_GIT`` is the path to the root of the yt git repository. -If you want to specify a specific unit test to run (and not run the entire -suite), you can do so by specifying the path of the test relative to the -``$YT_GIT/yt`` directory -- note that you strip off one yt more than you -normally would! For example, if you want to run the plot_window tests, you'd +If you only want to run tests in a specific file, you can do so by specifying the path of the test relative to the +``$YT_GIT/`` directory. For example, if you want to run the ``plot_window`` tests, you'd run: .. code-block:: bash - $ nosetests yt/visualization/tests/test_plotwindow.py + $ pytest yt/visualization/tests/test_plotwindow.py -Handling yt dependencies -^^^^^^^^^^^^^^^^^^^^^^^^ +Additionally, if you only want to run, say, ``test_all_fields`` in ``plot_window.py``, you can do so by running: -We attempt to make yt compatible with a wide variety of upstream software -versions. However, sometimes a specific version of a project that yt depends on -causes some breakage and must be blacklisted in the tests or a more -experimental project that yt depends on optionally might change sufficiently -that the yt community decides not to support an old version of that project. +.. code-block:: bash -To handle cases like this, the versions of upstream software projects installed -on the machines running the yt test suite are pinned to specific version -numbers that must be updated manually. This prevents breaking the yt tests when -a new version of an upstream dependency is released and allows us to manage -updates in upstream projects at our pace. + $ pytest yt/visualization/tests/test_plotwindow.py::test_all_fields -If you would like to add a new dependency for yt (even an optional dependency) -or would like to update a version of a yt dependency, you must edit the -``tests/test_requirements.txt`` file, this path is relative to the root of the -repository. This file contains an enumerated list of direct dependencies and -pinned version numbers. For new dependencies, simply append the name of the new -dependency to the end of the file, along with a pin to the latest version -number of the package. To update a package's version, simply update the version -number in the entry for that package. +See this `link `_ for more on how to invoke pytest. -Finally, we also run a set of tests with "minimal" dependencies installed. Please make sure any new tests you add that depend on an optional dependency are properly set up so that the test is not run if the dependency is not installed. If for some reason you need to update the listing of packages that are installed for the "minimal" dependency tests, you will need to edit ``tests/test_minimal_requirements.txt``. -How to Write Unit Tests -^^^^^^^^^^^^^^^^^^^^^^^ +Unit Test Tools +^^^^^^^^^^^^^^^ -yt provides several pieces of testing assistance, all in the ``yt.testing`` -module. Describing them in detail is somewhat outside the scope of this -document, as in some cases they belong to other packages. However, a few come +yt provides several tools to assist with writing unit tests. These tools all reside in the ``yt.testing`` +module. Describing them all in detail is somewhat outside the scope of this +document. However, a few come in handy: * :func:`~yt.testing.fake_random_ds` provides the ability to create a random dataset, with several fields and divided into several different - grids, that can be operated on. + grids. * :func:`~yt.testing.assert_equal` can operate on arrays. * :func:`~yt.testing.assert_almost_equal` can operate on arrays and accepts a relative allowable difference. * :func:`~yt.testing.assert_allclose_units` raises an error if two arrays are not equal up to a desired absolute or relative tolerance. This wraps numpy's - assert_allclose to correctly verify unit consistency as well. + ``testing.assert_allclose`` to correctly verify unit consistency as well. * :func:`~yt.testing.amrspace` provides the ability to create AMR grid structures. * :func:`~yt.testing.expand_keywords` provides the ability to iterate over many values for keywords. +How To Write New Unit Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + To create new unit tests: #. Create a new ``tests/`` directory next to the file containing the - functionality you want to test and add an empty ``__init__.py`` file to + machinery you want to test and add an empty ``__init__.py`` file to it. -#. Inside that directory, create a new python file prefixed with ``test_`` and +#. Inside this new ``tests/`` directory, create a new python file prefixed with ``test_`` and including the name of the functionality. -#. Inside that file, create one or more routines prefixed with ``test_`` that - accept no arguments. The test function should do some work that tests some +#. Inside this new ``test_`` file, create one or more routines prefixed with ``test_`` that + accept no arguments. +#. Each test function should do some work that tests some functionality and should also verify that the results are correct using assert statements or functions. -#. Tests can ``yield`` a tuple of the form ``function``, ``argument_one``, - ``argument_two``, etc. For example ``yield my_test, 'banana', 2.0`` would be - captured by nose and the ``my_test`` function will be run with the provided - arguments. -#. Use ``fake_random_ds`` to test on datasets, and be sure to test for - several combinations of ``nproc``, so that domain decomposition can be +#. If a dataset is needed, use ``fake_random_ds`` and be sure to test for + several combinations of ``nproc`` so that domain decomposition can be tested as well. -#. Test multiple combinations of options by using the - :func:`~yt.testing.expand_keywords` function, which will enable much - easier iteration over options. +#. To iterate over multiple options, or combinations of options, use the `pytest.mark.parametrize`_ decorator. For an example of how to write unit tests, look at the file ``yt/data_objects/tests/test_covering_grid.py``, which covers a great deal of @@ -148,358 +106,292 @@ functionality. Debugging failing tests ^^^^^^^^^^^^^^^^^^^^^^^ -When writing new tests, often one exposes bugs or writes a test incorrectly, +When writing new tests, one often exposes bugs or writes a test incorrectly, causing an exception to be raised or a failed test. To help debug issues like -this, ``nose`` can drop into a debugger whenever a test fails or raises an -exception. This can be accomplished by passing ``--pdb`` and ``--pdb-failures`` -to the ``nosetests`` executable. These options will drop into the pdb debugger -whenever an error is raised or a failure happens, respectively. Inside the +this, ``pytest`` can drop into a debugger whenever a test fails or raises an +exception. This can be accomplished by passing ``--pdb`` +to the ``pytest`` executable. Inside the debugger you can interactively print out variables and go up and down the call stack to determine the context for your failure or error. .. code-block:: bash - nosetests --pdb --pdb-failures + pytest --pdb In addition, one can debug more crudely using print statements. To do this, you can add print statements to the code as normal. However, the test runner will capture all print output by default. To ensure that output gets printed -to your terminal while the tests are running, pass ``-s`` to the ``nosetests`` +to your terminal while the tests are running, pass ``-s`` to the ``pytest`` executable. +.. code-block:: bash + + pytest -s + Lastly, to quickly debug a specific failing test, it is best to only run that one test during your testing session. This can be accomplished by explicitly -passing the name of the test function or class to ``nosetests``, as in the +passing the name of the test function or class to ``pytest``, as in the following example: .. code-block:: bash - $ nosetests yt.visualization.tests.test_plotwindow:TestSetWidth + $ pytest yt/visualization/tests/test_plotwindow.py::TestSetWidth -This nosetests invocation will only run the tests defined by the +This pytest invocation will only run the tests defined by the ``TestSetWidth`` class. Finally, to determine which test is failing while the tests are running, it helps to run the tests in "verbose" mode. This can be done by passing the ``-v`` option -to the ``nosetests`` executable. +to the ``pytest`` executable. + +.. code-block:: bash + + $ pytest -v -All of the above ``nosetests`` options can be combined. So, for example to run +All of the above ``pytest`` options can be combined. So, for example, to run the ``TestSetWidth`` tests with verbose output, letting the output of print statements come out on the terminal prompt, and enabling pdb debugging on errors or test failures, one would do: .. code-block:: bash - $ nosetests --pdb --pdb-failures -v -s yt.visualization.tests.test_plotwindow:TestSetWidth + $ pytest --pdb -v -s yt/visualization/tests/test_plotwindow.py::TestSetWidth + +More pytest options can be found by using the ``-h`` option + +.. code-block:: bash + + $ pytest -h .. _answer_testing: Answer Testing -------------- -What do Answer Tests Do +What Do Answer Tests Do ^^^^^^^^^^^^^^^^^^^^^^^ -Answer tests test **actual data**, and many operations on that data, to make -sure that answers don't drift over time. This is how we test frontends, as -opposed to operations, in yt. +Answer tests use `actual data `_ to test reading, writing, and various manipulations of that data. + +In order to ensure that each of these operations are performed correctly, yt has a set of yaml files containing the known correct results of having performed these operations. These files are called gold standard answer files. More generally, an answer file is a yaml file containing the results of having run the answer tests. + +When the answer tests are run, their output is compared to the gold standard answers to ensure that the results of operating on data do not change over time. .. _run_answer_testing: How to Run the Answer Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The very first step is to make a directory and copy over the data against which -you want to test. Next, add the config parameter ``test_data_dir`` pointing to -directory with the test data you want to test with, e.g.: +In order to run the answer tests locally: + +* Create a directory to hold the data required by the answer tests one wishes to run + +* Fill this directory with the required data + +Next, yt needs to be made aware of where it can find the data. This is done by setting the config parameter ``test_data_dir`` to the +directory with the test data. For example, .. code-block:: bash $ yt config set yt test_data_dir /Users/tomservo/src/yt-data -We use a number of real-world datasets for the tests that must be downloaded and -unzipped in the ``test_data_dir`` path you have set. The test datasets, can be -downloaded from https://yt-project.org/data/. We do not explicitly list the -datasets we use in the tests here because the list of necessary datasets changes -regularly, instead you should take a look at the tests you would like to run and -make sure that the necessary data files are downloaded before running the tests. +To run the answer tests locally, you must first generate a set of gold standard answer files using a "known good" version of yt. Once done, then update to the version of yt you want to test and +run the tests again so that the results can be compared to those contained in the gold standard answer files. -To run the answer tests, you must first generate a set of test answers locally -on a "known good" revision, then update to the revision you want to test, and -run the tests again using the locally stored answers. +As an example, let's focus on running the answer tests for the tipsy frontend. -Let's focus on running the answer tests for a single frontend. It's possible to -run the answer tests for **all** the frontends, but due to the large number of -test datasets we currently use this is not normally done except on the yt -project's contiguous integration server. +.. note:: + It's possible to run the answer tests for **all** the frontends, but due to the large number of test datasets we currently use this is not normally done except on the yt project's contiguous integration server. .. code-block:: bash $ cd $YT_GIT - $ nosetests --with-answer-testing --local --local-dir $HOME/Documents/test --answer-store --answer-name=local-tipsy yt.frontends.tipsy + $ pytest --with-answer-testing --answer-store --local-dir="$HOME/Documents/test" -k "TestTipsy" -This command will create a set of local answers from the tipsy frontend tests -and store them in ``$HOME/Documents/test`` (this can but does not have to be the -same directory as the ``test_data_dir`` configuration variable defined in your -``~/.config/yt/yt.toml`` file) in a file named ``local-tipsy``. To run the tipsy -frontend's answer tests using a different yt changeset, update to that -changeset, recompile if necessary, and run the tests using the following -command: +This command will run the tipsy answer tests and, because of the presence of the ``--answer-store`` option, save the results in a local gold standard answer file in ``$HOME/Documents/test``. -.. code-block:: bash +.. note:: + The path specified by ``--local-dir`` can, but does not have to be, the same directory as the ``test_data_dir`` configuration variable. + +The gold standard answer file will be named ``tipsy_answers_xyz.yaml``, where ``xyz`` denotes the version number of the gold standard answers. - $ nosetests --with-answer-testing --local --local-dir $HOME/Documents/test --answer-name=local-tipsy yt.frontends.tipsy +.. note:: + The answer version number is determined by the ``answer_version`` attribute of the class being tested (e.g., ``TestTipsy.answer_version``). -The results from a nose testing session are pretty straightforward to -understand, the results for each test are printed directly to STDOUT. If a test -passes, nose prints a period, F if a test fails, and E if the test encounters an -exception or errors out for some reason. Explicit descriptions for each test -are also printed if you pass ``-v`` to the ``nosetests`` executable. If you -want to also run tests for the 'big' datasets, then you will need to pass -``--answer-big-data`` to ``nosetests``. For example, to run the tests for the -OWLS frontend, do the following: +.. note:: + Changes made to yt sometimes result in known, expected changes to the way certain operations behave. This necessitates updating the gold standard answer files. This process is accomplished by changing the version number specified in each answer test class (e.g., ``TestTipsy.answer_version``). + +Once the gold standard answer file has been generated, update to the version of yt you wish to test, recompile if necessary, and run the tests using the following command: .. code-block:: bash - $ nosetests --with-answer-testing --local --local-dir $HOME/Documents/test --answer-store --answer-big-data yt.frontends.owls + $ pytest --with-answer-testing --local-dir="$HOME/Documents/test" -k "TestTipsy" + +The result of each test is printed to STDOUT. If a test passes, pytest prints a period. If a test fails or encounters an +exception or errors out for some reason, then an F is printed. Explicit descriptions for each test +are also printed if you pass ``-v`` to the ``pytest`` executable. Similar to the unit tests, the ``-s`` and ``--pdb`` options can be passed, as well. How to Write Answer Tests ^^^^^^^^^^^^^^^^^^^^^^^^^ -Tests can be added in the file ``yt/utilities/answer_testing/framework.py`` . -You can find examples there of how to write a test. Here is a trivial example: +To add a new answer test: -.. code-block:: python +#. Create a new directory called ``tests`` inside the directory where the component you want to test resides and add an empty ``__init__.py`` file to it. - #!python - class MaximumValueTest(AnswerTestingTest): - _type_name = "MaximumValue" - _attrs = ("field",) - - def __init__(self, ds_fn, field): - super(MaximumValueTest, self).__init__(ds_fn) - self.field = field - - def run(self): - v, c = self.ds.find_max(self.field) - result = np.empty(4, dtype="float64") - result[0] = v - result[1:] = c - return result - - def compare(self, new_result, old_result): - assert_equal(new_result, old_result) - -What this does is calculate the location and value of the maximum of a -field. It then puts that into the variable result, returns that from -``run`` and then in ``compare`` makes sure that all are exactly equal. - -To write a new test: - -* Subclass ``AnswerTestingTest`` -* Add the attributes ``_type_name`` (a string) and ``_attrs`` - (a tuple of strings, one for each attribute that defines the test -- - see how this is done for projections, for instance) -* Implement the two routines ``run`` and ``compare`` The first - should return a result and the second should compare a result to an old - result. Neither should yield, but instead actually return. If you need - additional arguments to the test, implement an ``__init__`` routine. -* Keep in mind that *everything* returned from ``run`` will be stored. So if - you are going to return a huge amount of data, please ensure that the test - only gets run for small data. If you want a fast way to measure something as - being similar or different, either an md5 hash (see the grid values test) or - a sum and std of an array act as good proxies. If you must store a large - amount of data for some reason, try serializing the data to a string - (e.g. using ``numpy.ndarray.dumps``), and then compressing the data stream - using ``zlib.compress``. -* Typically for derived values, we compare to 10 or 12 decimal places. - For exact values, we compare exactly. - -How To Write Answer Tests for a Frontend -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To add a new frontend answer test, first write a new set of tests for the data. -The Enzo example in ``yt/frontends/enzo/tests/test_outputs.py`` is -considered canonical. Do these things: - -* Create a new directory, ``tests`` inside the frontend's directory. - -* Create a new file, ``test_outputs.py`` in the frontend's ``tests`` - directory. - -* Create a new routine that operates similarly to the routines you can see - in Enzo's output tests. - - * This routine should test a number of different fields and data objects. - - * The test routine itself should be decorated with - ``@requires_ds(test_dataset_name)``. This decorator can accept the - argument ``big_data=True`` if the test is expensive. The - ``test_dataset_name`` should be a string containing the path you would pass - to the ``yt.load`` function. It does not need to be the full path to the - dataset, since the path will be automatically prepended with the location of - the test data directory. See :ref:`configuration-file` for more information - about the ``test_data-dir`` configuration option. - - * There are ``small_patch_amr`` and ``big_patch_amr`` routines that you can - yield from to execute a bunch of standard tests. In addition we have created - ``sph_answer`` which is more suited for particle SPH datasets. This is where - you should start, and then yield additional tests that stress the outputs in - whatever ways are necessary to ensure functionality. - -If you are adding to a frontend that has a few tests already, skip the first -two steps. +#. Create a new file in the ``tests`` directory that will hold the new answer tests. The name of the file should begin with ``test_``. -How to Write Image Comparison Tests -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +#. Create a new class whose name begins with ``Test`` (e.g., ``TestTipsy``). -We have a number of tests designed to compare images as part of yt. We make use -of some functionality from matplotlib to automatically compare images and detect -differences, if any. Image comparison tests are used in the plotting and volume -rendering machinery. +#. Decorate the class with ``pytest.mark.answer_test``. This decorator is used to tell pytest which tests are answer tests. -The easiest way to use the image comparison tests is to make use of the -``GenericImageTest`` class. This class takes three arguments: +.. note:: + Tests that do not have this decorator are considered to be unit tests. -* A dataset instance (e.g. something you load with ``yt.load`` or - ``data_dir_load``) -* A function the test machinery can call which will save an image to disk. The - test class will then find any images that get created and compare them with the - stored "correct" answer. -* An integer specifying the number of decimal places to use when comparing - images. A smaller number of decimal places will produce a less stringent test. - Matplotlib uses an L2 norm on the full image to do the comparison tests, so - this is not a pixel-by-pixel measure, and surprisingly large variations will - still pass the test if the strictness of the comparison is not high enough. +#. Add the following three attributes to the class: ``answer_file=None``, ``saved_hashes=None``, and ``answer_version=000``. These attributes are used by the ``hashing`` fixture (discussed below) to automate the creation of new answer files as well as facilitate the comparison to existing answers. + +#. Add methods to the class that test a number of different fields and data objects. -You *must* decorate your test function with ``requires_ds``, otherwise the -answer testing machinery will not be properly set up. +#. If these methods are performing calculations or data manipulation, they should store the result in a ``ndarray``, if possible. This array should be be added to the ``hashes`` (see below) dictionary like so: ``self.hashes.update(:)``, where ```` is the name of the function from ``yt/utilities/answer_testing/answer_tests.py`` that is being used and ```` is the ``ndarray`` holding the result -Here is an example test function: +If you are adding to a frontend that has tests already, simply add methods to the existing test class. + +There are several things that can make the test writing process easier: + +* ``yt/utilities/answer_testing/testing_utilities.py`` contains a large number of helper functions. +* Most frontends end up needing to test much of the same functionality as other frontends. As such, a list of functions that perform such work can be found in ``yt/utilities/answer_testing/answer_tests.py``. +* `Fixtures `_! You can find the set of fixtures that have already been built for yt in ``$YT_GIT/conftest.py``. If you need/want to add additional fixtures, please add them there. +* The `parametrize decorator `_ is extremely useful for performing iteration over various combinations of test parameters. It should be used whenever possible. + +Here is what a minimal example might look like for a new frontend: .. code-block:: python - from yt.utilities.answer_testing.framework import ( - GenericImageTest, - requires_ds, - data_dir_load, - ) + # Content of yt/frontends/new_frontend/tests/test_outputs.py + import pytest - from matplotlib import pyplot as plt + from yt.utilities.answer_testing.answer_tests import field_values + # Parameters to test with + ds1 = "my_first_dataset" + ds2 = "my_second_dataset" + field1 = ("Gas", "Density") + field2 = ("Gas", "Temperature") + obj1 = None + obj2 = ("sphere", ("c", (0.1, "unitary"))) - @requires_ds(my_ds) - def test_my_ds(): - ds = data_dir_load(my_ds) + @pytest.mark.answer_test + class TestNewFrontend: + answer_file = None + saved_hashes = None + answer_version = "000" - def create_image(filename_prefix): - plt.plot([1, 2], [1, 2]) - plt.savefig("%s_lineplot" % filename_prefix) + @pytest.mark.usefixtures("hashing") + @pytest.mark.parametrize("ds", [ds1, ds2], indirect=True) + @pytest.mark.parametrize("field", [field1, field2], indirect=True) + @pytest.mark.parametrize("dobj", [obj1, obj2], indirect=True) + def test_fields(self, ds, field, dobj): + self.hashes.update("field_values" : field_values(ds, field, dobj)) - test = GenericImageTest(ds, create_image, 12) +Answer test examples can be found in ``yt/frontends/enzo/tests/test_outputs.py``. - # this ensures the test has a unique key in the - # answer test storage file - test.prefix = "my_unique_name" +How to Write Image Comparison Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - # this ensures a nice test name in nose's output - test_my_ds.__name__ = test.description +Many of yt's operations involve creating and manipulating images. As such, we have a number of tests designed to compare images. These tests employ functionality from matplotlib to automatically compare images and detect +differences, if any. Image comparison tests are used in the plotting and volume +rendering machinery. - yield test +The easiest way to use the image comparison tests is to make use of the +``generic_image`` function. As an argument, this function takes a function the test machinery can call which will save an image to disk. The + test will then find any images that get created and compare them with the + stored "correct" answer. -.. note:: The inner function ``create_image`` can create any number of images, - as long as the corresponding filenames conform to the prefix. +Here is an example test function (from ``yt/visualization/tests/test_raw_field_slices.py``): -Another good example of an image comparison test is the -``PlotWindowAttributeTest`` defined in the answer testing framework and used in -``yt/visualization/tests/test_plotwindow.py``. This test shows how a new answer -test subclass can be used to programmatically test a variety of different methods -of a complicated class using the same test class. This sort of image comparison -test is more useful if you are finding yourself writing a ton of boilerplate -code to get your image comparison test working. The ``GenericImageTest`` is -more useful if you only need to do a one-off image comparison test. +.. code-block:: python -Enabling Answer Tests on Jenkins -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Before any code is added to or modified in the yt codebase, each incoming -changeset is run against all available unit and answer tests on our `continuous -integration server `_. While unit tests are -autodiscovered by `nose `_ itself, -answer tests require definition of which set of tests constitute to a given -answer. Configuration for the integration server is stored in -*tests/tests.yaml* in the main yt repository: - -.. code-block:: yaml - - answer_tests: - local_artio_000: - - yt/frontends/artio/tests/test_outputs.py - # ... - other_tests: - unittests: - - '-v' - - '-s' - -Each element under *answer_tests* defines answer name (*local_artio_000* in above -snippet) and specifies a list of files/classes/methods that will be validated -(*yt/frontends/artio/tests/test_outputs.py* in above snippet). On the testing -server it is translated to: + import pytest -.. code-block:: bash + import yt + from yt.utilities.answer_testing.answer_tests import generic_image + from yt.utilities.answer_testing.testing_utilities import data_dir_load, requires_ds - $ nosetests --with-answer-testing --local --local-dir ... --answer-big-data \ - --answer-name=local_artio_000 \ - yt/frontends/artio/tests/test_outputs.py + # Test data + raw_fields = "Laser/plt00015" -If the answer doesn't exist on the server yet, ``nosetests`` is run twice and -during first pass ``--answer-store`` is added to the commandline. + def compare(ds, field): + def slice_image(im_name): + sl = yt.SlicePlot(ds, "z", field) + sl.set_log("all", False) + image_file = sl.save(im_name) + return image_file -Updating Answers -~~~~~~~~~~~~~~~~ + gi = generic_image(slice_image) + # generic_image returns a list. In this case, there's only one entry, + # which is a np array with the data we want + return gi[0] -In order to regenerate answers for a particular set of tests it is sufficient to -change the answer name in *tests/tests.yaml* e.g.: + @pytest.mark.answer_test + @pytest.mark.usefixtures("temp_dir") + class TestRawFieldSlices: + answer_file = None + saved_hashes = None + answer_version = "000" -.. code-block:: diff + @pytest.mark.usefixtures("hashing") + @requires_ds(raw_fields) + def test_raw_field_slices(self, field): + ds = data_dir_load(raw_fields) + gi = compare(ds, field) + self.hashes.update({"generic_image": gi}) - --- a/tests/tests.yaml - +++ b/tests/tests.yaml - @@ -25,7 +25,7 @@ - - yt/analysis_modules/halo_finding/tests/test_rockstar.py - - yt/frontends/owls_subfind/tests/test_outputs.py +.. note:: + The inner function ``slice_image`` can create any number of images, as long as the corresponding filenames conform to the prefix. - - local_owls_000: - + local_owls_001: - - yt/frontends/owls/tests/test_outputs.py +Another good example of an image comparison test is the +``plot_window_attribute`` defined in the ``yt/utilities/answer_testing/answer_tests.py`` and used in +``yt/visualization/tests/test_plotwindow.py``. This sort of image comparison +test is more useful if you are finding yourself writing a ton of boilerplate +code to get your image comparison test working. The ``generic_image`` function is +more useful if you only need to do a one-off image comparison test. - local_pw_000: +Updating Answers +~~~~~~~~~~~~~~~~ -would regenerate answers for OWLS frontend. +In order to regenerate answers for a particular set of tests it is sufficient to +change the ``answer_version`` attribute in the desired test class. -When adding tests to an existing set of answers (like ``local_owls_000`` or ``local_varia_000``), +When adding tests to an existing set of answers (like ``local_owls_000.yaml`` or ``local_varia_000.yaml``), it is considered best practice to first submit a pull request adding the tests WITHOUT incrementing the version number. Then, allow the tests to run (resulting in "no old answer" errors for the missing answers). If no other failures are present, you can then increment the version number to regenerate the answers. This way, we can avoid accidentally covering up test breakages. -Adding New Answer Tests -~~~~~~~~~~~~~~~~~~~~~~~ +.. _handling_dependencies: -In order to add a new set of answer tests, it is sufficient to extend the -*answer_tests* list in *tests/tests.yaml* e.g.: +Handling yt dependencies +------------------------ -.. code-block:: diff +We attempt to make yt compatible with a wide variety of upstream software +versions. However, sometimes a specific version of a project that yt depends on +causes some breakage and must be blacklisted in the tests or a more +experimental project that yt depends on optionally might change sufficiently +that the yt community decides not to support an old version of that project. - --- a/tests/tests.yaml - +++ b/tests/tests.yaml - @@ -60,6 +60,10 @@ - - yt/analysis_modules/absorption_spectrum/tests/test_absorption_spectrum.py:test_absorption_spectrum_non_cosmo - - yt/analysis_modules/absorption_spectrum/tests/test_absorption_spectrum.py:test_absorption_spectrum_cosmo +To handle cases like this, the versions of upstream software projects installed +on the machines running the yt test suite are pinned to specific version +numbers that must be updated manually. This prevents breaking the yt tests when +a new version of an upstream dependency is released and allows us to manage +updates in upstream projects at our pace. + +If you would like to add a new dependency for yt (even an optional dependency) +or would like to update a version of a yt dependency, you must edit the +``tests/test_requirements.txt`` file, this path is relative to the root of the +repository. This file contains an enumerated list of direct dependencies and +pinned version numbers. For new dependencies, simply append the name of the new +dependency to the end of the file, along with a pin to the latest version +number of the package. To update a package's version, simply update the version +number in the entry for that package. - + local_gdf_000: - + - yt/frontends/gdf/tests/test_outputs.py - + - + - other_tests: - unittests: +Finally, we also run a set of tests with "minimal" dependencies installed. Please make sure any new tests you add that depend on an optional dependency are properly set up so that the test is not run if the dependency is not installed. If for some reason you need to update the listing of packages that are installed for the "minimal" dependency tests, you will need to edit ``tests/test_minimal_requirements.txt``. From d6278929e7cf121c6ecee282035733a5a18e0554 Mon Sep 17 00:00:00 2001 From: "Kacper Kowalik (Xarthisius)" Date: Fri, 19 Mar 2021 15:20:43 -0500 Subject: [PATCH 07/60] Use proper intersphinx links for pytest/numpy references --- doc/source/conf.py | 13 +++++++------ doc/source/developing/testing.rst | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 569e9aa79dd..b965ed8e86a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -251,15 +251,16 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "https://docs.python.org/3/": None, - "https://ipython.readthedocs.io/en/stable/": None, - "https://docs.scipy.org/doc/numpy/": None, - "https://matplotlib.org/stable/": None, - "https://docs.astropy.org/en/stable": None, - "https://pandas.pydata.org/pandas-docs/stable": None, + "python": ("https://docs.python.org/3/", None), + "ipython": ("https://ipython.readthedocs.io/en/stable/", None), + "numpy": ("https://numpy.org/doc/stable", None), + "matplotlib": ("https://matplotlib.org/stable/", None), + "astropy": ("https://docs.astropy.org/en/stable", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), "trident": ("https://trident.readthedocs.io/en/latest/", None), "yt_astro_analysis": ("https://yt-astro-analysis.readthedocs.io/en/latest/", None), "yt_attic": ("https://yt-attic.readthedocs.io/en/latest/", None), + "pytest": ("https://docs.pytest.org/en/stable", None), } if not on_rtd: diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 9584c999e85..d9ec39beccc 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -25,7 +25,7 @@ What Do Unit Tests Do Unit tests are tests that operate on some small piece of machinery and verify that the machinery works. In -practice, this means that we make assertions about a piece of machinery and then pytest runs the machinery, verifies that the assertions are true, and ensures that the code runs without crashing. An example of a unit test is ``test_all_fields`` in ``yt/fields/tests/test_fields.py``. +practice, this means that we make assertions about a piece of machinery and then pytest runs the machinery, verifies that the assertions are true, and ensures that the code runs without crashing. An example of a unit test is :func:`~yt.fields.tests.test_fields.test_all_fields`. How to Run the Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -73,7 +73,7 @@ in handy: relative allowable difference. * :func:`~yt.testing.assert_allclose_units` raises an error if two arrays are not equal up to a desired absolute or relative tolerance. This wraps numpy's - ``testing.assert_allclose`` to correctly verify unit consistency as well. + :py:func:`numpy.testing.assert_allclose` to correctly verify unit consistency as well. * :func:`~yt.testing.amrspace` provides the ability to create AMR grid structures. * :func:`~yt.testing.expand_keywords` provides the ability to iterate over @@ -97,7 +97,8 @@ To create new unit tests: #. If a dataset is needed, use ``fake_random_ds`` and be sure to test for several combinations of ``nproc`` so that domain decomposition can be tested as well. -#. To iterate over multiple options, or combinations of options, use the `pytest.mark.parametrize`_ decorator. +#. To iterate over multiple options, or combinations of options, + use the :ref:`@pytest.mark.parametrize <@pytest.mark.parametrize>` decorator. For an example of how to write unit tests, look at the file ``yt/data_objects/tests/test_covering_grid.py``, which covers a great deal of @@ -245,8 +246,9 @@ To add a new answer test: #. Decorate the class with ``pytest.mark.answer_test``. This decorator is used to tell pytest which tests are answer tests. -.. note:: - Tests that do not have this decorator are considered to be unit tests. + .. note:: + + Tests that do not have this decorator are considered to be unit tests. #. Add the following three attributes to the class: ``answer_file=None``, ``saved_hashes=None``, and ``answer_version=000``. These attributes are used by the ``hashing`` fixture (discussed below) to automate the creation of new answer files as well as facilitate the comparison to existing answers. @@ -260,8 +262,8 @@ There are several things that can make the test writing process easier: * ``yt/utilities/answer_testing/testing_utilities.py`` contains a large number of helper functions. * Most frontends end up needing to test much of the same functionality as other frontends. As such, a list of functions that perform such work can be found in ``yt/utilities/answer_testing/answer_tests.py``. -* `Fixtures `_! You can find the set of fixtures that have already been built for yt in ``$YT_GIT/conftest.py``. If you need/want to add additional fixtures, please add them there. -* The `parametrize decorator `_ is extremely useful for performing iteration over various combinations of test parameters. It should be used whenever possible. +* You can find the set of fixtures that have already been built for yt in ``$YT_GIT/conftest.py``. If you need/want to add additional fixtures, please add them there. See :std:doc:`fixture`. +* :ref:`parametrize` is extremely useful for performing iteration over various combinations of test parameters. It should be used whenever possible. Here is what a minimal example might look like for a new frontend: @@ -303,9 +305,7 @@ differences, if any. Image comparison tests are used in the plotting and volume rendering machinery. The easiest way to use the image comparison tests is to make use of the -``generic_image`` function. As an argument, this function takes a function the test machinery can call which will save an image to disk. The - test will then find any images that get created and compare them with the - stored "correct" answer. +``generic_image`` function. As an argument, this function takes a function the test machinery can call which will save an image to disk. The test will then find any images that get created and compare them with the stored "correct" answer. Here is an example test function (from ``yt/visualization/tests/test_raw_field_slices.py``): From 207d5b2980b09746dfb667c90e28bea783ed4233 Mon Sep 17 00:00:00 2001 From: Jared Coughlin Date: Mon, 22 Mar 2021 10:38:31 -0500 Subject: [PATCH 08/60] Addressed review comments. --- doc/source/developing/testing.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 9584c999e85..75903d97bd2 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -103,7 +103,7 @@ For an example of how to write unit tests, look at the file ``yt/data_objects/tests/test_covering_grid.py``, which covers a great deal of functionality. -Debugging failing tests +Debugging Failing Tests ^^^^^^^^^^^^^^^^^^^^^^^ When writing new tests, one often exposes bugs or writes a test incorrectly, @@ -189,7 +189,7 @@ In order to run the answer tests locally: * Fill this directory with the required data Next, yt needs to be made aware of where it can find the data. This is done by setting the config parameter ``test_data_dir`` to the -directory with the test data. For example, +directory with the test data downloaded from https://yt-project.org/data/. For example, .. code-block:: bash @@ -262,6 +262,8 @@ There are several things that can make the test writing process easier: * Most frontends end up needing to test much of the same functionality as other frontends. As such, a list of functions that perform such work can be found in ``yt/utilities/answer_testing/answer_tests.py``. * `Fixtures `_! You can find the set of fixtures that have already been built for yt in ``$YT_GIT/conftest.py``. If you need/want to add additional fixtures, please add them there. * The `parametrize decorator `_ is extremely useful for performing iteration over various combinations of test parameters. It should be used whenever possible. + * The use of this decorator allows pytest to write the names and values of the test parameters to the generated answer files, which can make debugging failing tests easier, since one can easily see exactly which combination of parameters were used for a given test. + * It is also possible to employ the ``requires_ds`` decorator to ensure that a test does not run unless a specific dataset is found, but not necessary. If the dataset is parametrized over, then the ``ds`` fixture found in the root ``conftest.py`` file performs the same check and marks the test as failed if the dataset isn't found. Here is what a minimal example might look like for a new frontend: @@ -370,7 +372,7 @@ the answers. This way, we can avoid accidentally covering up test breakages. .. _handling_dependencies: -Handling yt dependencies +Handling yt Dependencies ------------------------ We attempt to make yt compatible with a wide variety of upstream software @@ -394,4 +396,4 @@ dependency to the end of the file, along with a pin to the latest version number of the package. To update a package's version, simply update the version number in the entry for that package. -Finally, we also run a set of tests with "minimal" dependencies installed. Please make sure any new tests you add that depend on an optional dependency are properly set up so that the test is not run if the dependency is not installed. If for some reason you need to update the listing of packages that are installed for the "minimal" dependency tests, you will need to edit ``tests/test_minimal_requirements.txt``. +Finally, we also run a set of tests with "minimal" dependencies installed. When adding tests that depend on an optional dependency, you can wrap the test with the ``yt.testing.requires_module decorator`` to ensure it does not run during the minimal dependency tests (see yt/frontends/amrvac/tests/test_read_amrvac_namelist.py for a good example). If for some reason you need to update the listing of packages that are installed for the "minimal" dependency tests, you will need to edit ``tests/test_minimal_requirements.txt``. From d198f7c90b374eb1abf4716aa27105ae1a0ff41c Mon Sep 17 00:00:00 2001 From: Kacper Kowalik Date: Tue, 30 Mar 2021 08:27:17 -0500 Subject: [PATCH 09/60] fix: syntax error in docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- doc/source/developing/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 8a8f03b65e5..f93fc9910cd 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -295,7 +295,7 @@ Here is what a minimal example might look like for a new frontend: @pytest.mark.parametrize("field", [field1, field2], indirect=True) @pytest.mark.parametrize("dobj", [obj1, obj2], indirect=True) def test_fields(self, ds, field, dobj): - self.hashes.update("field_values" : field_values(ds, field, dobj)) + self.hashes.update({"field_values": field_values(ds, field, dobj)}) Answer test examples can be found in ``yt/frontends/enzo/tests/test_outputs.py``. From 170874c365271b202493672f4ae65398eb1060ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Mar 2021 13:29:15 +0000 Subject: [PATCH 10/60] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- doc/source/developing/testing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index f93fc9910cd..b10137040d2 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -284,6 +284,7 @@ Here is what a minimal example might look like for a new frontend: obj1 = None obj2 = ("sphere", ("c", (0.1, "unitary"))) + @pytest.mark.answer_test class TestNewFrontend: answer_file = None @@ -322,6 +323,7 @@ Here is an example test function (from ``yt/visualization/tests/test_raw_field_s # Test data raw_fields = "Laser/plt00015" + def compare(ds, field): def slice_image(im_name): sl = yt.SlicePlot(ds, "z", field) @@ -334,6 +336,7 @@ Here is an example test function (from ``yt/visualization/tests/test_raw_field_s # which is a np array with the data we want return gi[0] + @pytest.mark.answer_test @pytest.mark.usefixtures("temp_dir") class TestRawFieldSlices: From c65e0b27e5667b1d0b39195b4ebc9454955e62dc Mon Sep 17 00:00:00 2001 From: Jared Coughlin <30010597+jcoughlin11@users.noreply.github.com> Date: Wed, 12 May 2021 10:53:25 -0500 Subject: [PATCH 11/60] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- doc/source/developing/testing.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index b10137040d2..6d276da43ff 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -23,9 +23,11 @@ Unit Testing What Do Unit Tests Do ^^^^^^^^^^^^^^^^^^^^^ -Unit tests are tests that operate on some small piece of machinery and verify -that the machinery works. In -practice, this means that we make assertions about a piece of machinery and then pytest runs the machinery, verifies that the assertions are true, and ensures that the code runs without crashing. An example of a unit test is :func:`~yt.fields.tests.test_fields.test_all_fields`. +Unit tests are tests that operate on some small piece of code and verify that it behaves as intended. +In practice, this means that we write simple assertions (or ``assert`` statements) about results and +let pytest go through them. A test is considered a success when no error (in particular +``AssertionError``) occurs. +An example of such a test is :func:`~yt.fields.tests.test_fields.test_all_fields`. How to Run the Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^ From e86f634a98b8ef3d9a68463b32437a0234178573 Mon Sep 17 00:00:00 2001 From: Jared Coughlin Date: Wed, 12 May 2021 14:35:34 -0400 Subject: [PATCH 12/60] Addressed review comments. --- doc/source/developing/testing.rst | 120 +++++++++++++----------------- 1 file changed, 52 insertions(+), 68 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index b10137040d2..da611ab9c67 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -3,17 +3,16 @@ Testing ======= -yt includes a testing suite that one can run on the code base to assure that no +yt includes a testing suite that one can run on the code base to ensure that no breaks in functionality have occurred. This suite is based on the `pytest `_ testing framework and consists of two types of tests: -* Unit tests. These confirm that an isolated piece of functionality runs without failure for inputs with known correct outputs. +* Unit tests. These make sure that an small pieces of code run (or fail) as intended in predictable contexts. See :ref:`unit_testing`. -* Answer tests. These generate output from user-visible yt functions and compare this output against the output produced using an older, "known good", version of yt. +* Answer tests. These generate outputs from the user-facing yt API and compare them against the outputs produced using an older, "known good", version of yt. See :ref:`answer_testing`. These tests ensure consistency in results as development proceeds. -The testing suite should be run locally by developers to make sure they aren't -checking in any code that breaks existing functionality. Additionally, as a safeguard, the test suite is automatically run after each commit that is pushed to the yt repository on github. +We recommend that developers run tests locally on changed features when developing to help ensure that the new code does not break any existing functionality. To further this goal and ensure that changes do not propagate errors or have unintentional consequences on the rest of the codebase, the full test suite is run through our continuous integration (CI) servers. CI is run on push on open pull requests on a variety of computational platforms using Github Actions and a `continuous integration server `_ at the University of Illinois. The full test suite may take several hours to run, so we do not recommend running it locally. .. _unit_testing: @@ -23,9 +22,9 @@ Unit Testing What Do Unit Tests Do ^^^^^^^^^^^^^^^^^^^^^ -Unit tests are tests that operate on some small piece of machinery and verify -that the machinery works. In -practice, this means that we make assertions about a piece of machinery and then pytest runs the machinery, verifies that the assertions are true, and ensures that the code runs without crashing. An example of a unit test is :func:`~yt.fields.tests.test_fields.test_all_fields`. +Unit tests are tests that operate on some small piece of code and verify +that it behaves as intended. In +practice, this means that we write simple assertions (or ``assert`` statements) about results and then let pytest go through them. A test is considered a success when no error (in particular ``AssertionError``) occurs. How to Run the Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -48,36 +47,25 @@ run: $ pytest yt/visualization/tests/test_plotwindow.py -Additionally, if you only want to run, say, ``test_all_fields`` in ``plot_window.py``, you can do so by running: +from the yt source code root directory. + +Additionally, if you only want to run a specific test in a test file (rather than all of the tests contained in the file), such as, ``test_all_fields`` in ``plot_window.py``, you can do so by running: .. code-block:: bash $ pytest yt/visualization/tests/test_plotwindow.py::test_all_fields -See this `link `_ for more on how to invoke pytest. +from the yt source code rood directory. + +See the pytest documentation for more on how to `invoke pytest `_ and `select tests `_. Unit Test Tools ^^^^^^^^^^^^^^^ -yt provides several tools to assist with writing unit tests. These tools all reside in the ``yt.testing`` -module. Describing them all in detail is somewhat outside the scope of this -document. However, a few come -in handy: - -* :func:`~yt.testing.fake_random_ds` provides the ability to create a random - dataset, with several fields and divided into several different - grids. -* :func:`~yt.testing.assert_equal` can operate on arrays. -* :func:`~yt.testing.assert_almost_equal` can operate on arrays and accepts a - relative allowable difference. -* :func:`~yt.testing.assert_allclose_units` raises an error if two arrays are - not equal up to a desired absolute or relative tolerance. This wraps numpy's - :py:func:`numpy.testing.assert_allclose` to correctly verify unit consistency as well. -* :func:`~yt.testing.amrspace` provides the ability to create AMR grid - structures. -* :func:`~yt.testing.expand_keywords` provides the ability to iterate over - many values for keywords. +yt provides several helper functions and decorators to write unit tests. These tools all reside in the ``yt.testing`` +module. Describing them all in detail is outside the scope of this +document, as in some cases they belong to other packages. How To Write New Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -85,20 +73,22 @@ How To Write New Unit Tests To create new unit tests: #. Create a new ``tests/`` directory next to the file containing the - machinery you want to test and add an empty ``__init__.py`` file to + functionality you want to test and add an empty ``__init__.py`` file to it. + #. If a ``tests/`` directory already exists, there is no need to create a new one. #. Inside this new ``tests/`` directory, create a new python file prefixed with ``test_`` and - including the name of the functionality. + including the name of the functionality or source file being tested. + #. If a file testing the functionality you're interested in already exists, please add your tests to the existing file. #. Inside this new ``test_`` file, create one or more routines prefixed with ``test_`` that accept no arguments. #. Each test function should do some work that tests some functionality and should also verify that the results are correct using assert statements or functions. -#. If a dataset is needed, use ``fake_random_ds`` and be sure to test for +#. If a dataset is needed, use ``fake_random_ds``, ``fake_amr_ds``, or ``fake_particle_ds`` (the former two of which have a particle ``True``/``False`` option that may be utilized) and be sure to test for several combinations of ``nproc`` so that domain decomposition can be tested as well. #. To iterate over multiple options, or combinations of options, - use the :ref:`@pytest.mark.parametrize <@pytest.mark.parametrize>` decorator. + use the `@pytest.mark.parametrize decorator `_. For an example of how to write unit tests, look at the file ``yt/data_objects/tests/test_covering_grid.py``, which covers a great deal of @@ -106,28 +96,21 @@ functionality. Debugging Failing Tests ^^^^^^^^^^^^^^^^^^^^^^^ - When writing new tests, one often exposes bugs or writes a test incorrectly, -causing an exception to be raised or a failed test. To help debug issues like -this, ``pytest`` can drop into a debugger whenever a test fails or raises an -exception. This can be accomplished by passing ``--pdb`` -to the ``pytest`` executable. Inside the -debugger you can interactively print out variables and go up and down the call -stack to determine the context for your failure or error. - -.. code-block:: bash - pytest --pdb +causing an exception to be raised or a failed test. To help debug issues like +this, ``pytest`` can `drop into a debugger `_ whenever a test fails or raises an +exception. In addition, one can debug more crudely using print statements. To do this, you can add print statements to the code as normal. However, the test runner will capture all print output by default. To ensure that output gets printed -to your terminal while the tests are running, pass ``-s`` to the ``pytest`` +to your terminal while the tests are running, pass ``-s`` (which will disable stdout and stderr capturing) to the ``pytest`` executable. .. code-block:: bash - pytest -s + $ pytest -s Lastly, to quickly debug a specific failing test, it is best to only run that one test during your testing session. This can be accomplished by explicitly @@ -139,7 +122,7 @@ following example: $ pytest yt/visualization/tests/test_plotwindow.py::TestSetWidth This pytest invocation will only run the tests defined by the -``TestSetWidth`` class. +``TestSetWidth`` class. See the `pytest documentation `_ for more on the various ways to invoke pytest. Finally, to determine which test is failing while the tests are running, it helps to run the tests in "verbose" mode. This can be done by passing the ``-v`` option @@ -156,13 +139,13 @@ or test failures, one would do: .. code-block:: bash - $ pytest --pdb -v -s yt/visualization/tests/test_plotwindow.py::TestSetWidth + $ pytest yt/visualization/tests/test_plotwindow.py::TestSetWidth -v -s --pdb -More pytest options can be found by using the ``-h`` option +More pytest options can be found by using the ``--help`` flag .. code-block:: bash - $ pytest -h + $ pytest --help .. _answer_testing: @@ -172,11 +155,9 @@ Answer Testing What Do Answer Tests Do ^^^^^^^^^^^^^^^^^^^^^^^ -Answer tests use `actual data `_ to test reading, writing, and various manipulations of that data. - -In order to ensure that each of these operations are performed correctly, yt has a set of yaml files containing the known correct results of having performed these operations. These files are called gold standard answer files. More generally, an answer file is a yaml file containing the results of having run the answer tests. +Answer tests use `actual data `_ to test reading, writing, and various manipulations of that data. Answer tests are how we test frontends, as opposed to operations, in yt. -When the answer tests are run, their output is compared to the gold standard answers to ensure that the results of operating on data do not change over time. +In order to ensure that each of these operations are performed correctly, we store gold standard versions of yaml files called answer files. More generally, an answer file is a yaml file containing the results of having run the answer tests, which can be compared to a reference, enabling us to control that results do not drift over time. .. _run_answer_testing: @@ -185,21 +166,23 @@ How to Run the Answer Tests In order to run the answer tests locally: -* Create a directory to hold the data required by the answer tests one wishes to run +* Create a directory to hold the data you'll be using for the answer tests you'll be writing or the answer tests you'll be running. This directory should be outside the yt git repository in a place that is logical to where you would normally store data. -* Fill this directory with the required data +* Add folders of the required data to this directory. Other yt data, such as ``IsolatedGalaxy``, can be downloaded to this directory as well. -Next, yt needs to be made aware of where it can find the data. This is done by setting the config parameter ``test_data_dir`` to the +* Tell yt where it can find the data. This is done by setting the config parameter ``test_data_dir`` to the path of the directory with the test data downloaded from https://yt-project.org/data/. For example, .. code-block:: bash $ yt config set yt test_data_dir /Users/tomservo/src/yt-data -To run the answer tests locally, you must first generate a set of gold standard answer files using a "known good" version of yt. Once done, then update to the version of yt you want to test and -run the tests again so that the results can be compared to those contained in the gold standard answer files. +this should only need to be done once (unless you change where you're storing the data, in which case you'll need to repeat this step so yt looks in the right place). + +* Generate or obtain a set of gold standard answer files. In order to generate gold standard answer files, wwitch to a "known good" version of yt and then run the answer tests as described below. Once done, switch back to the version of yt you wish to test. +* Now you're ready to run the answer tests! -As an example, let's focus on running the answer tests for the tipsy frontend. +As an example, let's focus on running the answer tests for the tipsy frontend. Let's also assume that we need to generate a gold standard answer file. To do this, we first switch to a "known good" version of yt and run the following command from the top of the yt git directory (i.e., `$YT_GIT`) in order to generate the gold standard answer file: .. note:: It's possible to run the answer tests for **all** the frontends, but due to the large number of test datasets we currently use this is not normally done except on the yt project's contiguous integration server. @@ -209,27 +192,27 @@ As an example, let's focus on running the answer tests for the tipsy frontend. $ cd $YT_GIT $ pytest --with-answer-testing --answer-store --local-dir="$HOME/Documents/test" -k "TestTipsy" -This command will run the tipsy answer tests and, because of the presence of the ``--answer-store`` option, save the results in a local gold standard answer file in ``$HOME/Documents/test``. +The `--with-answer-testing` tells pytest that we want to run answer tests. Without this option, the unit tests will be run instead of the answer tests. The `--answer-store` option tells pytest to save the results produced by each test to a local gold standard answer file. Omitting this option is how we tell pytest to compare the results to a gold standard. The `--local-dir` option specifies where the gold standard answer file will be saved (or is already located, in the case that ``--answer-store`` is omitted). The `-k` option tells pytest that we only want to run tests whose name matches the given pattern. .. note:: - The path specified by ``--local-dir`` can, but does not have to be, the same directory as the ``test_data_dir`` configuration variable. - -The gold standard answer file will be named ``tipsy_answers_xyz.yaml``, where ``xyz`` denotes the version number of the gold standard answers. + The path specified by ``--local-dir`` can, but does not have to be, the same directory as the ``test_data_dir`` configuration variable. It is best practice to keep the data that serves as input to yt separate from the answers produced by yt's tests, however. .. note:: - The answer version number is determined by the ``answer_version`` attribute of the class being tested (e.g., ``TestTipsy.answer_version``). + The value given to the `-k` option (e.g., `"TestTipsy"`) is the name of the class containing the answer tests. You do not need to specify the path. + +The newly generated gold standard answer file will be named ``tipsy_answers_xyz.yaml``, where ``xyz`` denotes the version number of the gold standard answers. The answer version number is determined by the ``answer_version`` attribute of the class being tested (e.g., ``TestTipsy.answer_version``). .. note:: - Changes made to yt sometimes result in known, expected changes to the way certain operations behave. This necessitates updating the gold standard answer files. This process is accomplished by changing the version number specified in each answer test class (e.g., ``TestTipsy.answer_version``). + Changes made to yt sometimes result in known, expected changes to the way certain operations behave. This necessitates updating the gold standard answer files. This process is accomplished by changing the version number specified in each answer test class (e.g., ``TestTipsy.answer_version``). The answer version for each test class can be found as the attribute `answer_version` of that class. -Once the gold standard answer file has been generated, update to the version of yt you wish to test, recompile if necessary, and run the tests using the following command: +Once the gold standard answer file has been generated we switch back to the version of yt we want to test, recompile if necessary, and run the tests using the following command: .. code-block:: bash $ pytest --with-answer-testing --local-dir="$HOME/Documents/test" -k "TestTipsy" -The result of each test is printed to STDOUT. If a test passes, pytest prints a period. If a test fails or encounters an -exception or errors out for some reason, then an F is printed. Explicit descriptions for each test +The result of each test is printed to STDOUT. If a test passes, pytest prints a period. If a test fails, encounters an +exception, or errors out for some reason, then an F is printed. Explicit descriptions for each test are also printed if you pass ``-v`` to the ``pytest`` executable. Similar to the unit tests, the ``-s`` and ``--pdb`` options can be passed, as well. @@ -308,7 +291,7 @@ differences, if any. Image comparison tests are used in the plotting and volume rendering machinery. The easiest way to use the image comparison tests is to make use of the -``generic_image`` function. As an argument, this function takes a function the test machinery can call which will save an image to disk. The test will then find any images that get created and compare them with the stored "correct" answer. +``generic_image`` function. As an argument, this function takes a function the test machinery can call which will save an image to disk. The test will then find any images that get created and compare them with the stored "correct" answer. Here is an example test function (from ``yt/visualization/tests/test_raw_field_slices.py``): @@ -334,6 +317,7 @@ Here is an example test function (from ``yt/visualization/tests/test_raw_field_s gi = generic_image(slice_image) # generic_image returns a list. In this case, there's only one entry, # which is a np array with the data we want + assert len(gi) == 1 return gi[0] From b99c9a9c4f13b66f61ad49a1bbc97248440cf7d5 Mon Sep 17 00:00:00 2001 From: Jared Coughlin Date: Thu, 13 May 2021 15:02:11 -0400 Subject: [PATCH 13/60] Addresses review comments. --- doc/source/developing/testing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index da611ab9c67..8a58601b7f4 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -6,7 +6,7 @@ Testing yt includes a testing suite that one can run on the code base to ensure that no breaks in functionality have occurred. This suite is based on the `pytest `_ testing framework and consists of two types of tests: -* Unit tests. These make sure that an small pieces of code run (or fail) as intended in predictable contexts. See :ref:`unit_testing`. +* Unit tests. These make sure that small pieces of code run (or fail) as intended in predictable contexts. See :ref:`unit_testing`. * Answer tests. These generate outputs from the user-facing yt API and compare them against the outputs produced using an older, "known good", version of yt. See :ref:`answer_testing`. @@ -78,13 +78,13 @@ To create new unit tests: #. If a ``tests/`` directory already exists, there is no need to create a new one. #. Inside this new ``tests/`` directory, create a new python file prefixed with ``test_`` and including the name of the functionality or source file being tested. - #. If a file testing the functionality you're interested in already exists, please add your tests to the existing file. + #. If a file testing the functionality you're interested in already exists, please add your tests to the existing there. #. Inside this new ``test_`` file, create one or more routines prefixed with ``test_`` that accept no arguments. #. Each test function should do some work that tests some functionality and should also verify that the results are correct using assert statements or functions. -#. If a dataset is needed, use ``fake_random_ds``, ``fake_amr_ds``, or ``fake_particle_ds`` (the former two of which have a particle ``True``/``False`` option that may be utilized) and be sure to test for +#. If a dataset is needed, use ``fake_random_ds``, ``fake_amr_ds``, or ``fake_particle_ds`` (the former two of which have support for particles that may be utilized) and be sure to test for several combinations of ``nproc`` so that domain decomposition can be tested as well. #. To iterate over multiple options, or combinations of options, From c4ec7feacb62444df3e6e21ee54d89ae200f46e0 Mon Sep 17 00:00:00 2001 From: Jared Coughlin Date: Fri, 14 May 2021 10:59:46 -0400 Subject: [PATCH 14/60] Ran blacken-docs on testing.rst. --- doc/source/developing/testing.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 052d700dd12..0aacc63875b 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -308,7 +308,6 @@ Here is an example test function (from ``yt/visualization/tests/test_raw_field_s raw_fields = "Laser/plt00015" - def compare(ds, field): def slice_image(im_name): sl = yt.SlicePlot(ds, "z", field) From f4b88a78f3934da426059e6005fdada7233e4409 Mon Sep 17 00:00:00 2001 From: Jared Coughlin <30010597+jcoughlin11@users.noreply.github.com> Date: Thu, 20 May 2021 13:25:31 -0500 Subject: [PATCH 15/60] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- doc/source/developing/testing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 0aacc63875b..82f8ff53cb5 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -79,7 +79,7 @@ To create new unit tests: #. Inside this new ``tests/`` directory, create a new python file prefixed with ``test_`` and including the name of the functionality or source file being tested. #. If a file testing the functionality you're interested in already exists, please add your tests to the existing there. -#. Inside this new ``test_`` file, create one or more routines prefixed with ``test_`` that +#. Inside this new ``test_`` file, create functions prefixed with ``test_`` that accept no arguments. #. Each test function should do some work that tests some functionality and should also verify that the results are correct using From 6a487cdd9f315f873a7066f40560b5bfb8e91bd5 Mon Sep 17 00:00:00 2001 From: Jared Coughlin Date: Fri, 21 May 2021 10:40:10 -0500 Subject: [PATCH 16/60] Fixes rst syntax errors. --- doc/source/developing/testing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/developing/testing.rst b/doc/source/developing/testing.rst index 82f8ff53cb5..9f2298d0d88 100644 --- a/doc/source/developing/testing.rst +++ b/doc/source/developing/testing.rst @@ -182,7 +182,7 @@ this should only need to be done once (unless you change where you're storing th * Generate or obtain a set of gold standard answer files. In order to generate gold standard answer files, wwitch to a "known good" version of yt and then run the answer tests as described below. Once done, switch back to the version of yt you wish to test. * Now you're ready to run the answer tests! -As an example, let's focus on running the answer tests for the tipsy frontend. Let's also assume that we need to generate a gold standard answer file. To do this, we first switch to a "known good" version of yt and run the following command from the top of the yt git directory (i.e., `$YT_GIT`) in order to generate the gold standard answer file: +As an example, let's focus on running the answer tests for the tipsy frontend. Let's also assume that we need to generate a gold standard answer file. To do this, we first switch to a "known good" version of yt and run the following command from the top of the yt git directory (i.e., ``$YT_GIT``) in order to generate the gold standard answer file: .. note:: It's possible to run the answer tests for **all** the frontends, but due to the large number of test datasets we currently use this is not normally done except on the yt project's contiguous integration server. @@ -192,7 +192,7 @@ As an example, let's focus on running the answer tests for the tipsy frontend. L $ cd $YT_GIT $ pytest --with-answer-testing --answer-store --local-dir="$HOME/Documents/test" -k "TestTipsy" -The `--with-answer-testing` tells pytest that we want to run answer tests. Without this option, the unit tests will be run instead of the answer tests. The `--answer-store` option tells pytest to save the results produced by each test to a local gold standard answer file. Omitting this option is how we tell pytest to compare the results to a gold standard. The `--local-dir` option specifies where the gold standard answer file will be saved (or is already located, in the case that ``--answer-store`` is omitted). The `-k` option tells pytest that we only want to run tests whose name matches the given pattern. +The ``--with-answer-testing`` tells pytest that we want to run answer tests. Without this option, the unit tests will be run instead of the answer tests. The ``--answer-store`` option tells pytest to save the results produced by each test to a local gold standard answer file. Omitting this option is how we tell pytest to compare the results to a gold standard. The ``--local-dir`` option specifies where the gold standard answer file will be saved (or is already located, in the case that ``--answer-store`` is omitted). The ``-k`` option tells pytest that we only want to run tests whose name matches the given pattern. .. note:: The path specified by ``--local-dir`` can, but does not have to be, the same directory as the ``test_data_dir`` configuration variable. It is best practice to keep the data that serves as input to yt separate from the answers produced by yt's tests, however. From 9b37dc780a36a473c4f8f0d03ed2d83e59de1a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 10 Jun 2021 15:45:06 +0200 Subject: [PATCH 17/60] make CoordinateHandler an abstract class --- yt/geometry/coordinates/coordinate_handler.py | 32 ++++++++++++------- .../coordinates/cylindrical_coordinates.py | 3 ++ .../coordinates/geographic_coordinates.py | 3 ++ .../coordinates/spherical_coordinates.py | 3 ++ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/yt/geometry/coordinates/coordinate_handler.py b/yt/geometry/coordinates/coordinate_handler.py index 8d72900ed35..81e34b4447d 100644 --- a/yt/geometry/coordinates/coordinate_handler.py +++ b/yt/geometry/coordinates/coordinate_handler.py @@ -1,3 +1,4 @@ +import abc import weakref from numbers import Number @@ -61,7 +62,7 @@ def validate_sequence_width(width, ds, unit=None): ) -class CoordinateHandler: +class CoordinateHandler(abc.ABC): name = None def __init__(self, ds, ordering): @@ -70,38 +71,46 @@ def __init__(self, ds, ordering): def setup_fields(self): # This should return field definitions for x, y, z, r, theta, phi - raise NotImplementedError + pass + @abc.abstractmethod def pixelize(self, dimension, data_source, field, bounds, size, antialias=True): # This should *actually* be a pixelize call, not just returning the # pixelizer - raise NotImplementedError + pass + @abc.abstractmethod def pixelize_line(self, field, start_point, end_point, npoints): - raise NotImplementedError + pass def distance(self, start, end): p1 = self.convert_to_cartesian(start) p2 = self.convert_to_cartesian(end) return np.sqrt(((p1 - p2) ** 2.0).sum()) + @abc.abstractmethod def convert_from_cartesian(self, coord): - raise NotImplementedError + pass + @abc.abstractmethod def convert_to_cartesian(self, coord): - raise NotImplementedError + pass + @abc.abstractmethod def convert_to_cylindrical(self, coord): - raise NotImplementedError + pass + @abc.abstractmethod def convert_from_cylindrical(self, coord): - raise NotImplementedError + pass + @abc.abstractmethod def convert_to_spherical(self, coord): - raise NotImplementedError + pass + @abc.abstractmethod def convert_from_spherical(self, coord): - raise NotImplementedError + pass _data_projection = None @@ -194,8 +203,9 @@ def y_axis(self): return ya @property + @abc.abstractproperty def period(self): - raise NotImplementedError + pass def sanitize_depth(self, depth): if is_sequence(depth): diff --git a/yt/geometry/coordinates/cylindrical_coordinates.py b/yt/geometry/coordinates/cylindrical_coordinates.py index 357887cf9ef..c8914bd16f2 100644 --- a/yt/geometry/coordinates/cylindrical_coordinates.py +++ b/yt/geometry/coordinates/cylindrical_coordinates.py @@ -165,6 +165,9 @@ def pixelize( # Pixelizing along a cylindrical surface is a bit tricky raise NotImplementedError + def pixelize_line(self, field, start_point, end_point, npoints): + raise NotImplementedError + def _ortho_pixelize( self, data_source, field, bounds, size, antialias, dim, periodic ): diff --git a/yt/geometry/coordinates/geographic_coordinates.py b/yt/geometry/coordinates/geographic_coordinates.py index 535970c851a..37604091211 100644 --- a/yt/geometry/coordinates/geographic_coordinates.py +++ b/yt/geometry/coordinates/geographic_coordinates.py @@ -245,6 +245,9 @@ def pixelize( else: raise NotImplementedError + def pixelize_line(self, field, start_point, end_point, npoints): + raise NotImplementedError + def _ortho_pixelize( self, data_source, field, bounds, size, antialias, dimension, periodic ): diff --git a/yt/geometry/coordinates/spherical_coordinates.py b/yt/geometry/coordinates/spherical_coordinates.py index 69b9abed1cb..e5c11e687e7 100644 --- a/yt/geometry/coordinates/spherical_coordinates.py +++ b/yt/geometry/coordinates/spherical_coordinates.py @@ -164,6 +164,9 @@ def pixelize( else: raise NotImplementedError + def pixelize_line(self, field, start_point, end_point, npoints): + raise NotImplementedError + def _ortho_pixelize( self, data_source, field, bounds, size, antialias, dim, periodic ): From c14a09c1a13b28d17b0f780b2c87fd611df79083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Fri, 9 Jul 2021 10:48:32 +0200 Subject: [PATCH 18/60] fix: a more specific and robust implementation for AthenaDataset._is_valid --- yt/frontends/athena/data_structures.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/yt/frontends/athena/data_structures.py b/yt/frontends/athena/data_structures.py index d3bd9dd1353..e8235a07974 100644 --- a/yt/frontends/athena/data_structures.py +++ b/yt/frontends/athena/data_structures.py @@ -1,4 +1,5 @@ import os +import re import weakref import numpy as np @@ -632,12 +633,18 @@ def _parse_parameter_file(self): @classmethod def _is_valid(cls, filename, *args, **kwargs): - try: - if "vtk" in filename: - return True - except Exception: - pass - return False + if not filename.endswith(".vtk"): + return False + + with open(filename, "rb") as fh: + if not re.match(b"# vtk DataFile Version \\d\\.\\d\n", fh.readline(256)): + return False + if not re.match( + b"(CONSERVED|PRIMITIVE) vars at time= .*, level= \\d, domain= \\d\n", + fh.readline(256), + ): + return False + return True @property def _skip_cache(self): From 9555edcb89966ecaa4e6958f750f980d5e234e98 Mon Sep 17 00:00:00 2001 From: Britton Smith Date: Tue, 20 Jul 2021 10:37:15 +0100 Subject: [PATCH 19/60] Update yt/geometry/particle_geometry_handler.py Co-authored-by: Meagan Lang --- yt/geometry/particle_geometry_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 8c5c8fc3346..a8c13d200f2 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -210,6 +210,7 @@ def _initialize_coarse_index(self): self.regions._coarse_index_data_file(pos, hsml, data_file.file_id) pb.finish() self.regions.masks = self.comm.mpi_allreduce(self.regions.masks, op="sum") + self.regions.particle_counts = self.comm.mpi_allreduce(self.regions.particle_counts, op="sum") for data_file in self.data_files: self.regions._set_coarse_index_data_file(data_file.file_id) self.regions.find_collisions_coarse() From 683f76defd9f29702d0f6b0a8ce144e42ff2d2da Mon Sep 17 00:00:00 2001 From: Britton Smith Date: Tue, 20 Jul 2021 13:18:35 +0100 Subject: [PATCH 20/60] Black formatting. --- yt/geometry/particle_geometry_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt/geometry/particle_geometry_handler.py b/yt/geometry/particle_geometry_handler.py index 8a69838540b..52cab7a6609 100644 --- a/yt/geometry/particle_geometry_handler.py +++ b/yt/geometry/particle_geometry_handler.py @@ -219,7 +219,9 @@ def _initialize_coarse_index(self): self.regions._coarse_index_data_file(pos, hsml, data_file.file_id) pb.finish() self.regions.masks = self.comm.mpi_allreduce(self.regions.masks, op="sum") - self.regions.particle_counts = self.comm.mpi_allreduce(self.regions.particle_counts, op="sum") + self.regions.particle_counts = self.comm.mpi_allreduce( + self.regions.particle_counts, op="sum" + ) for data_file in self.data_files: self.regions._set_coarse_index_data_file(data_file.file_id) self.regions.find_collisions_coarse() From 6ba2938390e806fa0cf806134851b2b21579bd33 Mon Sep 17 00:00:00 2001 From: Matthew Turk Date: Tue, 20 Jul 2021 12:09:44 -0500 Subject: [PATCH 21/60] Return if no string to read/write. --- yt/utilities/lib/ewah_bool_wrap.pyx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/yt/utilities/lib/ewah_bool_wrap.pyx b/yt/utilities/lib/ewah_bool_wrap.pyx index 284b63f7ee5..d2352d2e82b 100644 --- a/yt/utilities/lib/ewah_bool_wrap.pyx +++ b/yt/utilities/lib/ewah_bool_wrap.pyx @@ -574,9 +574,11 @@ cdef class FileBitmasks: cdef np.uint64_t nrefn, mi1 nrefn = mi1 = 0 # Write string to string stream + if len(s) == 0: return 1 ss.write(s, len(s)) # Read keys and refinement arrays ewah_keys[0].read(ss,1) + if ss.eof(): return 1 ewah_refn[0].read(ss,1) # Read and check number of refined cells ss.read( (&nrefn), sizeof(nrefn)) @@ -585,6 +587,7 @@ cdef class FileBitmasks: # Loop over refined cells for _ in range(nrefn): ss.read( (&mi1), sizeof(mi1)) + if ss.eof(): return 1 ewah_coll[0][mi1].read(ss,1) # or... #mi1_ewah.read(ss,1) @@ -1226,11 +1229,15 @@ cdef class BoolArrayCollection: cdef np.uint64_t nrefn, mi1 nrefn = mi1 = 0 # Write string to string stream + if len(s) == 0: return 1 ss.write(s, len(s)) # Read keys and refinement arrays + if ss.eof(): return 1 ewah_keys[0].read(ss,1) + if ss.eof(): return 1 ewah_refn[0].read(ss,1) # Read and check number of refined cells + if ss.eof(): return 1 ss.read( (&nrefn), sizeof(nrefn)) if nrefn != ewah_refn[0].numberOfOnes(): raise Exception("Error in read. File indicates {} refinements, but bool array has {}.".format(nrefn,ewah_refn[0].numberOfOnes())) From 8f88ad6412cf1dc071237c1d48c23a6c22e11f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 20 Jul 2021 23:57:55 +0200 Subject: [PATCH 22/60] BUG: fix a broken error message for yt.load in case of FileNotFoundError --- yt/sample_data/api.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/yt/sample_data/api.py b/yt/sample_data/api.py index fc951365fb7..34557b51df2 100644 --- a/yt/sample_data/api.py +++ b/yt/sample_data/api.py @@ -143,7 +143,7 @@ def _get_test_data_dir_path(): return Path.cwd() -def lookup_on_disk_data(fn): +def lookup_on_disk_data(fn) -> Path: """ Look for data file/dir on disk. @@ -163,15 +163,14 @@ def lookup_on_disk_data(fn): return path alt_path = _get_test_data_dir_path() / fn - err_msg = f"No such file or directory: '{fn}'." - if not alt_path.parent.is_dir(): - raise FileNotFoundError(err_msg) + if alt_path.exists(): + return alt_path - if not alt_path.exists(): + err_msg = f"No such file or directory: '{fn}'." + if alt_path.parent.is_dir() and alt_path != path: err_msg += f"\n(Also tried '{alt_path}')." - raise FileNotFoundError(err_msg) - return alt_path + raise FileNotFoundError(err_msg) @lru_cache(maxsize=128) From c91b918fe1886bf3645fe16c97d934a93c031122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Tue, 27 Jul 2021 18:30:52 +0200 Subject: [PATCH 23/60] fix a missing abc.abstractmethod decorator --- yt/geometry/coordinates/coordinate_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt/geometry/coordinates/coordinate_handler.py b/yt/geometry/coordinates/coordinate_handler.py index 81e34b4447d..f5e9e522eb9 100644 --- a/yt/geometry/coordinates/coordinate_handler.py +++ b/yt/geometry/coordinates/coordinate_handler.py @@ -69,6 +69,7 @@ def __init__(self, ds, ordering): self.ds = weakref.proxy(ds) self.axis_order = ordering + @abc.abstractmethod def setup_fields(self): # This should return field definitions for x, y, z, r, theta, phi pass From 8ff8549353a11283f44014ff9752b71f85a24bb1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 16:35:15 +0000 Subject: [PATCH 24/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.0 → v2.23.1](https://github.com/asottile/pyupgrade/compare/v2.23.0...v2.23.1) - [github.com/PyCQA/isort: 5.9.2 → 5.9.3](https://github.com/PyCQA/isort/compare/5.9.2...5.9.3) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 962e8202ceb..a39d2ad717a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: - id: no-commit-to-branch args: [--branch, main] - repo: https://github.com/asottile/pyupgrade - rev: v2.23.0 + rev: v2.23.1 hooks: - id: pyupgrade args: [--py36-plus] @@ -38,7 +38,7 @@ repos: - id: black language_version: python3 - repo: https://github.com/PyCQA/isort - rev: '5.9.2' + rev: '5.9.3' hooks: - id: isort name: isort (python) From ba6a2a84f52b9c45e5c1c4dbf642611d6c588697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 5 Aug 2021 15:09:10 +0200 Subject: [PATCH 25/60] DOC: fix two mistakes in installing.rst --- doc/source/installing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/installing.rst b/doc/source/installing.rst index 59aeec7f87c..5edaee32992 100644 --- a/doc/source/installing.rst +++ b/doc/source/installing.rst @@ -29,7 +29,7 @@ or pip, and we will *assume* that you do so in an isolated environment. Also note that each yt release supports a limited range of Python versions. Here's a summary for most recent releases -+------------+------------+----------------------------------+ ++------------+------------+----------------+-----------------+ | yt release | Python 2.7 | Python3 min | Python3 max | +============+============+================+=================+ | 4.1.x | no | 3.7 (expected) | 3.10 (expected) | @@ -120,7 +120,7 @@ yt as by appending ``[full]`` to the target name when calling pip, i.e., Testing Your Installation +++++++++++++++++++++++++ -To test to make sure everything is installed properly, try running yt at +To make sure everything is installed properly, try running yt at the command line: .. code-block:: bash From 99133ac79b5239aa0e9c0a1d1a851fa2d33ed768 Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Fri, 6 Aug 2021 14:04:34 -0700 Subject: [PATCH 26/60] fixed issue #3449 --- yt/visualization/plot_modifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index fccb0e7fd95..601235171a5 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -78,7 +78,7 @@ def _project_coords(self, plot, coord): if len(coord) == 3: if not isinstance(coord, YTArray): coord = plot.data.ds.arr(coord, "code_length") - coord.convert_to_units("code_length") + coord.to("code_length") ax = plot.data.axis # if this is an on-axis projection or slice, then # just grab the appropriate 2 coords for the on-axis view From 94ef5598c0cbab9ffb99332bb5487c3686b944c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Aug 2021 16:34:00 +0000 Subject: [PATCH 27/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.1 → v2.23.3](https://github.com/asottile/pyupgrade/compare/v2.23.1...v2.23.3) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a39d2ad717a..08b2b192075 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: - id: no-commit-to-branch args: [--branch, main] - repo: https://github.com/asottile/pyupgrade - rev: v2.23.1 + rev: v2.23.3 hooks: - id: pyupgrade args: [--py36-plus] From 66923a23ff2d54ffa729072bb473fdaad4574db4 Mon Sep 17 00:00:00 2001 From: atmyers Date: Mon, 9 Aug 2021 16:32:40 -0700 Subject: [PATCH 28/60] Add support for global_startindex != 0 to covering_grid --- yt/data_objects/construction_data_containers.py | 7 ++++++- yt/data_objects/static_output.py | 1 + yt/frontends/boxlib/data_structures.py | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index cfc544394f2..29dc93d36c5 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -650,10 +650,15 @@ def __init__( self._use_pbar = use_pbar self.global_startindex = np.rint( (self.left_edge - self.ds.domain_left_edge) / self.dds - ).astype("int64") + ).astype("int64")+self.ds.domain_offset self._setup_data_source() self.get_data(fields) + def get_global_startindex(self): + r"""Get the global start index of the covering grid. + """ + return self.global_startindex + def to_xarray(self, fields=None): r"""Export this fixed-resolution object to an xarray Dataset diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 601d2988341..11611f2615a 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -136,6 +136,7 @@ class Dataset(abc.ABC): domain_left_edge = MutableAttribute(True) domain_right_edge = MutableAttribute(True) domain_dimensions = MutableAttribute(True) + domain_offset = np.array([0, 0, 0]) _periodicity = MutableAttribute() _force_periodicity = False diff --git a/yt/frontends/boxlib/data_structures.py b/yt/frontends/boxlib/data_structures.py index 77a6da6aa3e..4ff6a06177c 100644 --- a/yt/frontends/boxlib/data_structures.py +++ b/yt/frontends/boxlib/data_structures.py @@ -881,6 +881,9 @@ def _parse_header_file(self): stop = np.array(root_space[1].split(","), dtype="int64") dd = np.ones(3, dtype="int64") dd[: self.dimensionality] = stop - start + 1 + do = np.zeros(3, dtype="int64") + do[: self.dimensionality] = start + self.domain_offset = do self.domain_dimensions = dd # Skip timesteps per level From a9ac30aa23218dd0be856069ebc60c1edefec14c Mon Sep 17 00:00:00 2001 From: Navaneeth Suresh Date: Tue, 10 Aug 2021 19:13:38 +0530 Subject: [PATCH 29/60] replace cmap with ndarray for refined_count --- yt/geometry/particle_oct_container.pyx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yt/geometry/particle_oct_container.pyx b/yt/geometry/particle_oct_container.pyx index d0f547bbbe7..27e329675b1 100644 --- a/yt/geometry/particle_oct_container.pyx +++ b/yt/geometry/particle_oct_container.pyx @@ -723,14 +723,12 @@ cdef class ParticleBitmap: cdef int axiter[3][2] cdef np.float64_t axiterv[3][2] cdef CoarseRefinedSets coarse_refined_map - cdef cmap[np.uint64_t, np.uint64_t] refined_count cdef np.uint64_t nfully_enclosed = 0, n_calls = 0 mi1_max = (1 << self.index_order1) - 1 mi2_max = (1 << self.index_order2) - 1 cdef np.uint64_t max_mi1_elements = 1 << (3*self.index_order1) cdef np.uint64_t max_mi2_elements = 1 << (3*self.index_order2) - for i in range(max_mi1_elements): - refined_count[i] = 0 + cdef np.ndarray[np.uint64_t, ndim=1] refined_count = np.zeros(max_mi1_elements, dtype="uint64") # Copy things from structure (type cast) for i in range(3): LE[i] = self.left_edge[i] From 0bd0210595cf9496d3d197e5c26a42ae28e5f4c9 Mon Sep 17 00:00:00 2001 From: atmyers Date: Tue, 10 Aug 2021 09:35:36 -0700 Subject: [PATCH 30/60] reformat --- yt/data_objects/construction_data_containers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index 29dc93d36c5..ca5f8f53895 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -648,15 +648,17 @@ def __init__( self.right_edge = self.left_edge + self.ActiveDimensions * self.dds self._num_ghost_zones = num_ghost_zones self._use_pbar = use_pbar - self.global_startindex = np.rint( - (self.left_edge - self.ds.domain_left_edge) / self.dds - ).astype("int64")+self.ds.domain_offset + self.global_startindex = ( + np.rint((self.left_edge - self.ds.domain_left_edge) / self.dds).astype( + "int64" + ) + + self.ds.domain_offset + ) self._setup_data_source() self.get_data(fields) def get_global_startindex(self): - r"""Get the global start index of the covering grid. - """ + r"""Get the global start index of the covering grid.""" return self.global_startindex def to_xarray(self, fields=None): From d8d39e546c352d6d7879276728aff37606b4fee6 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Tue, 10 Aug 2021 09:36:53 -0700 Subject: [PATCH 31/60] Update yt/data_objects/static_output.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/data_objects/static_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 11611f2615a..9fa5b3995eb 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -136,7 +136,7 @@ class Dataset(abc.ABC): domain_left_edge = MutableAttribute(True) domain_right_edge = MutableAttribute(True) domain_dimensions = MutableAttribute(True) - domain_offset = np.array([0, 0, 0]) + domain_offset = np.zeros(3, dtype="int64") _periodicity = MutableAttribute() _force_periodicity = False From cc28fa27dcc75c85d71c5a8ca889414777599978 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Tue, 10 Aug 2021 09:36:59 -0700 Subject: [PATCH 32/60] Update yt/frontends/boxlib/data_structures.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/frontends/boxlib/data_structures.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/yt/frontends/boxlib/data_structures.py b/yt/frontends/boxlib/data_structures.py index 4ff6a06177c..f43cc37d3a0 100644 --- a/yt/frontends/boxlib/data_structures.py +++ b/yt/frontends/boxlib/data_structures.py @@ -881,9 +881,7 @@ def _parse_header_file(self): stop = np.array(root_space[1].split(","), dtype="int64") dd = np.ones(3, dtype="int64") dd[: self.dimensionality] = stop - start + 1 - do = np.zeros(3, dtype="int64") - do[: self.dimensionality] = start - self.domain_offset = do + self.domain_offset[: self.dimensionality] = start self.domain_dimensions = dd # Skip timesteps per level From d0f6ab0896c24ec0be5a5918e998a1e00bf43b49 Mon Sep 17 00:00:00 2001 From: atmyers Date: Tue, 10 Aug 2021 09:40:21 -0700 Subject: [PATCH 33/60] add comment about domain_offset --- yt/data_objects/static_output.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 9fa5b3995eb..8a8f67db184 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -136,6 +136,8 @@ class Dataset(abc.ABC): domain_left_edge = MutableAttribute(True) domain_right_edge = MutableAttribute(True) domain_dimensions = MutableAttribute(True) + # the point in index space "domain_left_edge" maps to + # not necessarily (0, 0, 0) domain_offset = np.zeros(3, dtype="int64") _periodicity = MutableAttribute() _force_periodicity = False From 01825ec7bea8d5934482de2b5dcfc5b07f9186f3 Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Tue, 10 Aug 2021 21:42:43 -0700 Subject: [PATCH 34/60] needed to save output of .to --- yt/visualization/plot_modifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index 601235171a5..27b50fc762d 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -78,7 +78,7 @@ def _project_coords(self, plot, coord): if len(coord) == 3: if not isinstance(coord, YTArray): coord = plot.data.ds.arr(coord, "code_length") - coord.to("code_length") + coord = coord.to("code_length") ax = plot.data.axis # if this is an on-axis projection or slice, then # just grab the appropriate 2 coords for the on-axis view From ed9c674902079379cbdbd7d38897fd994522a85b Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Wed, 11 Aug 2021 15:13:40 -0700 Subject: [PATCH 35/60] added line annotation test --- .../tests/test_line_annotation_unit.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 yt/visualization/tests/test_line_annotation_unit.py diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py new file mode 100644 index 00000000000..5a5ccd1358f --- /dev/null +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -0,0 +1,27 @@ +import yt +import numpy as np + + +def test_ds_arr_invariance_under_projection_plot(): + data_array = np.random.random((10, 10, 10)) + bbox = np.array([[-100, 100], [-100, 100], [-100, 100]]) + data = {("gas", "density"): (data_array, "g*cm**(-3)")} + ds = yt.load_uniform_grid(data, data_array.shape, length_unit="kpc", bbox=bbox) + + start_source = np.array((0, 0, -0.5)) + end_source = np.array((0, 0, 0.5)) + start = ds.arr(start_source, "unitary") + end = ds.arr(end_source, "unitary") + + start_i = start.copy() + end_i = end.copy() + + p = yt.ProjectionPlot(ds, 0, "number_density") + p.annotate_line(start, end) + p.show() + + # for lack of a unyt.testing.assert_unit_array_equal function + np.testing.assert_array_equal(start_i, start) + assert start_i.units == start + np.testing.assert_array_equal(end_i, end) + assert end_i.units == end \ No newline at end of file From 9f8fb0da7c7c2dce53a38f0e89a27da0fed75aa4 Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Wed, 11 Aug 2021 15:39:42 -0700 Subject: [PATCH 36/60] fixed unit assertion test --- yt/visualization/tests/test_line_annotation_unit.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py index 5a5ccd1358f..a35a678f143 100644 --- a/yt/visualization/tests/test_line_annotation_unit.py +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -18,10 +18,9 @@ def test_ds_arr_invariance_under_projection_plot(): p = yt.ProjectionPlot(ds, 0, "number_density") p.annotate_line(start, end) - p.show() # for lack of a unyt.testing.assert_unit_array_equal function np.testing.assert_array_equal(start_i, start) - assert start_i.units == start + assert start_i.units == start.units np.testing.assert_array_equal(end_i, end) - assert end_i.units == end \ No newline at end of file + assert end_i.units == end.units \ No newline at end of file From a91a98b141e4b7937bad2e3c36972ff60bddf7df Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Wed, 11 Aug 2021 15:53:25 -0700 Subject: [PATCH 37/60] renamed 'coord' to 'coord_copy' to not overwrite, added comments --- yt/visualization/plot_modifications.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index 27b50fc762d..782d4d5c922 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -77,8 +77,10 @@ def _project_coords(self, plot, coord): """ if len(coord) == 3: if not isinstance(coord, YTArray): - coord = plot.data.ds.arr(coord, "code_length") - coord = coord.to("code_length") + coord_copy = plot.data.ds.arr(coord, "code_length") + #coord is being copied so that if the user has a unyt_array already + #we don't change the user's version + coord_copy = coord.to("code_length") ax = plot.data.axis # if this is an on-axis projection or slice, then # just grab the appropriate 2 coords for the on-axis view @@ -87,7 +89,7 @@ def _project_coords(self, plot, coord): plot.data.ds.coordinates.x_axis[ax], plot.data.ds.coordinates.y_axis[ax], ) - coord = (coord[xi], coord[yi]) + ret_coord = (coord_copy[xi], coord_copy[yi]) # if this is an off-axis project or slice (ie cutting plane) # we have to calculate where the data coords fall in the projected @@ -95,20 +97,20 @@ def _project_coords(self, plot, coord): elif ax == 4: # transpose is just to get [[x1,x2,...],[y1,y2,...],[z1,z2,...]] # in the same order as plot.data.center for array arithmetic - coord_vectors = coord.transpose() - plot.data.center + coord_vectors = coord_copy.transpose() - plot.data.center x = np.dot(coord_vectors, plot.data.orienter.unit_vectors[1]) y = np.dot(coord_vectors, plot.data.orienter.unit_vectors[0]) # Transpose into image coords. Due to VR being not a # right-handed coord system - coord = (y, x) + ret_coord = (y, x) else: - raise ValueError("Object being plot must have a `data.axis` defined") + raise ValueError("Object being plotted must have a `data.axis` defined") # if the position is already two-coords, it is expected to be # in the proper projected orientation else: raise ValueError("'data' coordinates must be 3 dimensions") - return coord + return ret_coord def _convert_to_plot(self, plot, coord, offset=True): """ From 1f3312d98f3b803f85dbefbcdd7a3582f1d72929 Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Wed, 11 Aug 2021 16:02:31 -0700 Subject: [PATCH 38/60] restored 'p.show()' --- yt/visualization/tests/test_line_annotation_unit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py index a35a678f143..32b882d839b 100644 --- a/yt/visualization/tests/test_line_annotation_unit.py +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -18,7 +18,8 @@ def test_ds_arr_invariance_under_projection_plot(): p = yt.ProjectionPlot(ds, 0, "number_density") p.annotate_line(start, end) - + p.show() + # for lack of a unyt.testing.assert_unit_array_equal function np.testing.assert_array_equal(start_i, start) assert start_i.units == start.units From 7b056f6e6759f1a0626d6b8bbf20d07a94aa2795 Mon Sep 17 00:00:00 2001 From: atmyers Date: Wed, 11 Aug 2021 16:13:56 -0700 Subject: [PATCH 39/60] Don't get domain dimensions from cparam file. --- yt/frontends/boxlib/data_structures.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/yt/frontends/boxlib/data_structures.py b/yt/frontends/boxlib/data_structures.py index 77a6da6aa3e..3a16cf56633 100644 --- a/yt/frontends/boxlib/data_structures.py +++ b/yt/frontends/boxlib/data_structures.py @@ -743,19 +743,7 @@ def _parse_cparams(self): param, vals = (s.strip() for s in line.split("=")) except ValueError: continue - if param == "amr.n_cell": - vals = self.domain_dimensions = np.array(vals.split(), dtype="int32") - - # For 1D and 2D simulations in BoxLib usually only the relevant - # dimensions have a specified number of zones, but yt requires - # domain_dimensions to have three elements, with 1 in the additional - # slots if we're not in 3D, so append them as necessary. - - if self.dimensionality == 1: - vals = self.domain_dimensions = np.array([vals[0], 1, 1]) - elif self.dimensionality == 2: - vals = self.domain_dimensions = np.array([vals[0], vals[1], 1]) - elif param == "amr.ref_ratio": + if param == "amr.ref_ratio": vals = self.refine_by = int(vals[0]) elif param == "Prob.lo_bc": vals = tuple(p == "1" for p in vals.split()) From 66ca34e431014fa527f68b54b37b97d8f1c40020 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Thu, 12 Aug 2021 05:14:06 -0700 Subject: [PATCH 40/60] Update yt/data_objects/static_output.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/data_objects/static_output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 8a8f67db184..61d7f51e00c 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -136,8 +136,8 @@ class Dataset(abc.ABC): domain_left_edge = MutableAttribute(True) domain_right_edge = MutableAttribute(True) domain_dimensions = MutableAttribute(True) - # the point in index space "domain_left_edge" maps to - # not necessarily (0, 0, 0) + # the point in index space "domain_left_edge" doesn't necessarily + # map to (0, 0, 0) domain_offset = np.zeros(3, dtype="int64") _periodicity = MutableAttribute() _force_periodicity = False From c9f6254489e0e99a61a0da98c546d0d4354d1306 Mon Sep 17 00:00:00 2001 From: atmyers Date: Thu, 12 Aug 2021 05:20:33 -0700 Subject: [PATCH 41/60] trim trailing whitespace --- yt/data_objects/static_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/data_objects/static_output.py b/yt/data_objects/static_output.py index 61d7f51e00c..3b1c255f207 100644 --- a/yt/data_objects/static_output.py +++ b/yt/data_objects/static_output.py @@ -136,7 +136,7 @@ class Dataset(abc.ABC): domain_left_edge = MutableAttribute(True) domain_right_edge = MutableAttribute(True) domain_dimensions = MutableAttribute(True) - # the point in index space "domain_left_edge" doesn't necessarily + # the point in index space "domain_left_edge" doesn't necessarily # map to (0, 0, 0) domain_offset = np.zeros(3, dtype="int64") _periodicity = MutableAttribute() From 3eaf48b6333c9a979a6806cddc1e652467f10075 Mon Sep 17 00:00:00 2001 From: atmyers Date: Thu, 12 Aug 2021 06:09:58 -0700 Subject: [PATCH 42/60] remove unused dictionaries from definitions.py --- yt/frontends/boxlib/definitions.py | 47 ------------------------------ 1 file changed, 47 deletions(-) diff --git a/yt/frontends/boxlib/definitions.py b/yt/frontends/boxlib/definitions.py index 187c8f9a536..e69de29bb2d 100644 --- a/yt/frontends/boxlib/definitions.py +++ b/yt/frontends/boxlib/definitions.py @@ -1,47 +0,0 @@ -# TODO: get rid of enzo parameters we do not need -parameterDict = { - "CosmologyCurrentRedshift": float, - "CosmologyComovingBoxSize": float, - "CosmologyOmegaMatterNow": float, - "CosmologyOmegaLambdaNow": float, - "CosmologyHubbleConstantNow": float, - "CosmologyInitialRedshift": float, - "DualEnergyFormalismEta1": float, - "DualEnergyFormalismEta2": float, - "MetaDataString": str, - "HydroMethod": int, - "DualEnergyFormalism": int, - "InitialTime": float, - "ComovingCoordinates": int, - "DensityUnits": float, - "LengthUnits": float, - "LengthUnit": float, - "TemperatureUnits": float, - "TimeUnits": float, - "GravitationalConstant": float, - "Gamma": float, - "MultiSpecies": int, - "CompilerPrecision": str, - "CurrentTimeIdentifier": int, - "RefineBy": int, - "BoundaryConditionName": str, - "TopGridRank": int, - "TopGridDimensions": int, - "EOSSoundSpeed": float, - "EOSType": int, - "NumberOfParticleAttributes": int, -} - - -# converts the Orion inputs file name to the Enzo/yt name expected -# throughout the code. key is Orion name, value is Enzo/yt equivalent -orion2enzoDict = { - "amr.n_cell": "TopGridDimensions", - "materials.gamma": "Gamma", - "amr.ref_ratio": "RefineBy", - "castro.use_comoving": "ComovingCoordinates", - "castro.redshift_in": "CosmologyInitialRedshift", - "comoving_OmL": "CosmologyOmegaLambdaNow", - "comoving_OmM": "CosmologyOmegaMatterNow", - "comoving_h": "CosmologyHubbleConstantNow", -} From 77938dbf45faa15b93bd0dbcfc566c99b3c3a3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 16 Aug 2021 13:23:47 +0200 Subject: [PATCH 43/60] DOC: use absolute links instead of relative path for quickstart guard linking with README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4c3a4efd182..023f9b9c05a 100644 --- a/README.md +++ b/README.md @@ -63,12 +63,12 @@ can be found on [the project's website](https://yt-project.org/docs/dev/installi yt is designed to provide meaningful analysis of data. We have some Quickstart example notebooks in the repository: - * [Introduction](doc/source/quickstart/1\)_Introduction.ipynb) - * [Data Inspection](doc/source/quickstart/2\)_Data_Inspection.ipynb) - * [Simple Visualization](doc/source/quickstart/3\)_Simple_Visualization.ipynb) - * [Data Objects and Time Series](doc/source/quickstart/4\)_Data_Objects_and_Time_Series.ipynb) - * [Derived Fields and Profiles](doc/source/quickstart/5\)_Derived_Fields_and_Profiles.ipynb) - * [Volume Rendering](doc/source/quickstart/6\)_Volume_Rendering.ipynb) + * [Introduction](https://github.com/yt-project/yt/tree/main/doc/source/quickstart/1\)_Introduction.ipynb) + * [Data Inspection](https://github.com/yt-project/yt/tree/main/doc/source/quickstart/2\)_Data_Inspection.ipynb) + * [Simple Visualization](https://github.com/yt-project/yt/tree/main/doc/source/quickstart/3\)_Simple_Visualization.ipynb) + * [Data Objects and Time Series](https://github.com/yt-project/yt/tree/main/doc/source/quickstart/4\)_Data_Objects_and_Time_Series.ipynb) + * [Derived Fields and Profiles](https://github.com/yt-project/yt/tree/main/doc/source/quickstart/5\)_Derived_Fields_and_Profiles.ipynb) + * [Volume Rendering](https://github.com/yt-project/yt/tree/main/doc/source/quickstart/6\)_Volume_Rendering.ipynb) If you'd like to try these online, you can visit our [yt Hub](https://hub.yt/) and run a notebook next to some of our example data. From 22c367be810c6c0e278e946b87383b6bbb17faa7 Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Tue, 17 Aug 2021 11:50:38 -0700 Subject: [PATCH 44/60] added arg to test/excluded test from nose/fixed bug if not passed ytarray --- nose_unit.cfg | 2 +- tests/tests.yaml | 1 + yt/visualization/plot_modifications.py | 3 ++- yt/visualization/tests/test_line_annotation_unit.py | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nose_unit.cfg b/nose_unit.cfg index 6a91cedd562..a34b709f157 100644 --- a/nose_unit.cfg +++ b/nose_unit.cfg @@ -6,5 +6,5 @@ nologcapture=1 verbosity=2 where=yt with-timer=1 -ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py) +ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py) exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF diff --git a/tests/tests.yaml b/tests/tests.yaml index 49a5915f6f7..621bad06615 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -182,6 +182,7 @@ other_tests: - '--ignore=test_outputs_pytest' - '--ignore-files=test_load_errors.py' - '--ignore-files=test_commons.py' + - '--ignore-files=test_line_annotation_unit.py' - '--ignore-files=test_load_sample.py' - '--ignore-files=test_field_access_pytest.py' - '--ignore-files=test_ambiguous_fields.py' diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index 782d4d5c922..8fbfd2ff7ff 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -80,7 +80,8 @@ def _project_coords(self, plot, coord): coord_copy = plot.data.ds.arr(coord, "code_length") #coord is being copied so that if the user has a unyt_array already #we don't change the user's version - coord_copy = coord.to("code_length") + else: + coord_copy = coord.to("code_length") ax = plot.data.axis # if this is an on-axis projection or slice, then # just grab the appropriate 2 coords for the on-axis view diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py index 32b882d839b..160cc3eaf59 100644 --- a/yt/visualization/tests/test_line_annotation_unit.py +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -2,7 +2,7 @@ import numpy as np -def test_ds_arr_invariance_under_projection_plot(): +def test_ds_arr_invariance_under_projection_plot(tmp_path): data_array = np.random.random((10, 10, 10)) bbox = np.array([[-100, 100], [-100, 100], [-100, 100]]) data = {("gas", "density"): (data_array, "g*cm**(-3)")} @@ -18,7 +18,7 @@ def test_ds_arr_invariance_under_projection_plot(): p = yt.ProjectionPlot(ds, 0, "number_density") p.annotate_line(start, end) - p.show() + p.save(tmp_path) # for lack of a unyt.testing.assert_unit_array_equal function np.testing.assert_array_equal(start_i, start) From 22744698fdcd66536a4666b4f187bffcccb8c2a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Aug 2021 19:28:41 +0000 Subject: [PATCH 45/60] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/visualization/plot_modifications.py | 4 ++-- yt/visualization/tests/test_line_annotation_unit.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index 8fbfd2ff7ff..ac124e72274 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -78,8 +78,8 @@ def _project_coords(self, plot, coord): if len(coord) == 3: if not isinstance(coord, YTArray): coord_copy = plot.data.ds.arr(coord, "code_length") - #coord is being copied so that if the user has a unyt_array already - #we don't change the user's version + # coord is being copied so that if the user has a unyt_array already + # we don't change the user's version else: coord_copy = coord.to("code_length") ax = plot.data.axis diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py index 160cc3eaf59..77edd969579 100644 --- a/yt/visualization/tests/test_line_annotation_unit.py +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -1,6 +1,7 @@ -import yt import numpy as np +import yt + def test_ds_arr_invariance_under_projection_plot(tmp_path): data_array = np.random.random((10, 10, 10)) @@ -19,9 +20,9 @@ def test_ds_arr_invariance_under_projection_plot(tmp_path): p = yt.ProjectionPlot(ds, 0, "number_density") p.annotate_line(start, end) p.save(tmp_path) - + # for lack of a unyt.testing.assert_unit_array_equal function np.testing.assert_array_equal(start_i, start) assert start_i.units == start.units np.testing.assert_array_equal(end_i, end) - assert end_i.units == end.units \ No newline at end of file + assert end_i.units == end.units From bf01a180eb1740b474115d80faefa74f31bb6891 Mon Sep 17 00:00:00 2001 From: Clayton Strawn <33767568+claytonstrawn@users.noreply.github.com> Date: Tue, 17 Aug 2021 12:38:31 -0700 Subject: [PATCH 46/60] Update yt/visualization/plot_modifications.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit move comment to else clause Co-authored-by: Clément Robert --- yt/visualization/plot_modifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/visualization/plot_modifications.py b/yt/visualization/plot_modifications.py index ac124e72274..9fc3e140e78 100644 --- a/yt/visualization/plot_modifications.py +++ b/yt/visualization/plot_modifications.py @@ -78,9 +78,9 @@ def _project_coords(self, plot, coord): if len(coord) == 3: if not isinstance(coord, YTArray): coord_copy = plot.data.ds.arr(coord, "code_length") - # coord is being copied so that if the user has a unyt_array already - # we don't change the user's version else: + # coord is being copied so that if the user has a unyt_array already + # we don't change the user's version coord_copy = coord.to("code_length") ax = plot.data.axis # if this is an on-axis projection or slice, then From 719964576acb08ee606f431445645a193eba7f33 Mon Sep 17 00:00:00 2001 From: Clayton Strawn Date: Tue, 17 Aug 2021 13:36:30 -0700 Subject: [PATCH 47/60] changed import statement in test --- yt/visualization/tests/test_line_annotation_unit.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py index 77edd969579..65d83af18b9 100644 --- a/yt/visualization/tests/test_line_annotation_unit.py +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -1,13 +1,12 @@ +from yt.loaders import load_uniform_grid +from yt.visualization.plot_window import ProjectionPlot import numpy as np -import yt - - def test_ds_arr_invariance_under_projection_plot(tmp_path): data_array = np.random.random((10, 10, 10)) bbox = np.array([[-100, 100], [-100, 100], [-100, 100]]) data = {("gas", "density"): (data_array, "g*cm**(-3)")} - ds = yt.load_uniform_grid(data, data_array.shape, length_unit="kpc", bbox=bbox) + ds = load_uniform_grid(data, data_array.shape, length_unit="kpc", bbox=bbox) start_source = np.array((0, 0, -0.5)) end_source = np.array((0, 0, 0.5)) @@ -17,7 +16,7 @@ def test_ds_arr_invariance_under_projection_plot(tmp_path): start_i = start.copy() end_i = end.copy() - p = yt.ProjectionPlot(ds, 0, "number_density") + p = ProjectionPlot(ds, 0, "number_density") p.annotate_line(start, end) p.save(tmp_path) From 86a7d925231cc97698d5c5e4d6f4cb0a64eb2c3c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Aug 2021 20:45:47 +0000 Subject: [PATCH 48/60] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- yt/visualization/tests/test_line_annotation_unit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt/visualization/tests/test_line_annotation_unit.py b/yt/visualization/tests/test_line_annotation_unit.py index 65d83af18b9..8a49cc3ce02 100644 --- a/yt/visualization/tests/test_line_annotation_unit.py +++ b/yt/visualization/tests/test_line_annotation_unit.py @@ -1,6 +1,8 @@ +import numpy as np + from yt.loaders import load_uniform_grid from yt.visualization.plot_window import ProjectionPlot -import numpy as np + def test_ds_arr_invariance_under_projection_plot(tmp_path): data_array = np.random.random((10, 10, 10)) From ab38db04803ec2e2a61ef0b728b3418935915aeb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:33:20 +0000 Subject: [PATCH 49/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.23.3 → v2.24.0](https://github.com/asottile/pyupgrade/compare/v2.23.3...v2.24.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 08b2b192075..87c5101ae18 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: - id: no-commit-to-branch args: [--branch, main] - repo: https://github.com/asottile/pyupgrade - rev: v2.23.3 + rev: v2.24.0 hooks: - id: pyupgrade args: [--py36-plus] From d24952daeff3899f195eb27ad3d525e3fadec213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 25 Aug 2021 07:41:48 +0200 Subject: [PATCH 50/60] MNT: set upper limit to firefly-vis version in [full] target as a workaround a backward incompatibility breakage --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b4888bdb35f..099a5743acf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,7 +74,7 @@ full = astropy>=4.0.1,<5.0.0 f90nml>=1.1.2 fastcache~=1.0.2 - firefly-vis>=2.0.0 + firefly-vis>=2.0.0,<2.0.3 #https://github.com/ageller/Firefly/issues/62 glueviz~=0.13.3 h5py>=3.1.0,<4.0.0 libconf~=1.0.1 From 9b6b81eb9c76f770a619e37ca1684246484fd3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 25 Aug 2021 17:12:58 +0200 Subject: [PATCH 51/60] MNT: add button to manually run bleeding edge CI job --- .github/workflows/bleeding-edge.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bleeding-edge.yaml b/.github/workflows/bleeding-edge.yaml index dae94349cd8..f5a624e2e8a 100644 --- a/.github/workflows/bleeding-edge.yaml +++ b/.github/workflows/bleeding-edge.yaml @@ -13,6 +13,7 @@ on: schedule: # run this every day at 3 am UTC - cron: '0 3 * * *' + workflow_dispatch: jobs: build: From bec60e2b138bce668c1c0539a64f84a0f7154a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 25 Aug 2021 20:04:21 +0200 Subject: [PATCH 52/60] BUG: fix a broken call to warnings.warn --- yt/data_objects/construction_data_containers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yt/data_objects/construction_data_containers.py b/yt/data_objects/construction_data_containers.py index ca5f8f53895..4c814cd2862 100644 --- a/yt/data_objects/construction_data_containers.py +++ b/yt/data_objects/construction_data_containers.py @@ -1412,8 +1412,7 @@ def _fill_fields(self, fields): warnings.warn( "Something went wrong during field computation. " "This is likely due to missing ghost-zones support " - "in class %s", - self.ds.__class__, + f"in class {type(self.ds)}", category=RuntimeWarning, ) mylog.debug("Caught %d runtime errors.", runtime_errors_count) From 3b1170362eccab2bf96af8b5cb07b437a5e79413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Thu, 26 Aug 2021 15:16:43 +0200 Subject: [PATCH 53/60] BUG: fix mpl data path handling for Windows --- yt/visualization/plot_container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yt/visualization/plot_container.py b/yt/visualization/plot_container.py index 9b1f1d5f055..062b1ed57ba 100644 --- a/yt/visualization/plot_container.py +++ b/yt/visualization/plot_container.py @@ -257,7 +257,9 @@ def __init__(self, data_source, figure_size, fontsize): self.figure_size = float(figure_size[0]), float(figure_size[1]) else: self.figure_size = float(figure_size) - font_path = matplotlib.get_data_path() + "/fonts/ttf/STIXGeneral.ttf" + font_path = os.path.join( + matplotlib.get_data_path(), "fonts", "ttf", "STIXGeneral.ttf" + ) self._font_properties = FontProperties(size=fontsize, fname=font_path) self._font_color = None self._xlabel = None From 76cdbb7479bbce642f28ccbecbfa158d8c79eed3 Mon Sep 17 00:00:00 2001 From: John ZuHone Date: Sat, 28 Aug 2021 22:14:44 -0400 Subject: [PATCH 54/60] The order for the y-axis was wrong here --- yt/visualization/fits_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/visualization/fits_image.py b/yt/visualization/fits_image.py index 6c895694642..bc65793586c 100644 --- a/yt/visualization/fits_image.py +++ b/yt/visualization/fits_image.py @@ -848,7 +848,7 @@ def sanitize_fits_unit(unit): # This list allows one to determine which axes are the # correct axes of the image in a right-handed coordinate # system depending on which axis is sliced or projected -axis_wcs = [[1, 2], [0, 2], [0, 1]] +axis_wcs = [[1, 2], [2, 0], [0, 1]] def construct_image(ds, axis, data_source, center, image_res, width, length_unit): From 0f91188976954bb84b7e6032b17de607480d3fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Sun, 29 Aug 2021 19:36:13 +0200 Subject: [PATCH 55/60] ENH: loosen header matching for Athena vtk files to allow compatibility with single variable 'vtk outputs' --- yt/frontends/athena/data_structures.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/yt/frontends/athena/data_structures.py b/yt/frontends/athena/data_structures.py index e8235a07974..7e3bb2a3606 100644 --- a/yt/frontends/athena/data_structures.py +++ b/yt/frontends/athena/data_structures.py @@ -639,10 +639,19 @@ def _is_valid(cls, filename, *args, **kwargs): with open(filename, "rb") as fh: if not re.match(b"# vtk DataFile Version \\d\\.\\d\n", fh.readline(256)): return False - if not re.match( - b"(CONSERVED|PRIMITIVE) vars at time= .*, level= \\d, domain= \\d\n", - fh.readline(256), + if ( + re.search( + b"at time= .*, level= \\d, domain= \\d\n", + fh.readline(256), + ) + is None ): + # vtk Dumps headers start with either "CONSERVED vars" or "PRIMITIVE vars", + # while vtk output headers start with "Really cool Athena data", but + # we will consider this an implementation detail and not attempt to + # match it exactly here. + # See Athena's user guide for reference + # https://princetonuniversity.github.io/Athena-Cversion/AthenaDocsUGbtk return False return True From d03b618372b99b6028c11e50f8365d265bf1a9e2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 16:37:45 +0000 Subject: [PATCH 56/60] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v2.24.0 → v2.25.0](https://github.com/asottile/pyupgrade/compare/v2.24.0...v2.25.0) - [github.com/psf/black: 21.7b0 → 21.8b0](https://github.com/psf/black/compare/21.7b0...21.8b0) - [github.com/asottile/blacken-docs: v1.10.0 → v1.11.0](https://github.com/asottile/blacken-docs/compare/v1.10.0...v1.11.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87c5101ae18..46bfbd1f75d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,12 +28,12 @@ repos: - id: no-commit-to-branch args: [--branch, main] - repo: https://github.com/asottile/pyupgrade - rev: v2.24.0 + rev: v2.25.0 hooks: - id: pyupgrade args: [--py36-plus] - repo: https://github.com/psf/black - rev: 21.7b0 + rev: 21.8b0 hooks: - id: black language_version: python3 @@ -55,7 +55,7 @@ repos: flake8-2020==1.6.0, ] - repo: https://github.com/asottile/blacken-docs - rev: v1.10.0 + rev: v1.11.0 hooks: - id: blacken-docs additional_dependencies: [black==20.8b1] From 9485a53a9bdb169d3b04c7899c4addd598b2c263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 30 Aug 2021 18:43:07 +0200 Subject: [PATCH 57/60] MNT: manually sync black version for docs --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46bfbd1f75d..3fd1422928a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: rev: v1.11.0 hooks: - id: blacken-docs - additional_dependencies: [black==20.8b1] + additional_dependencies: [black==21.8b0] - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.9.0 hooks: From feda7b1e692f2c5e9033ae32d57670f9f02be139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 1 Sep 2021 09:08:37 +0200 Subject: [PATCH 58/60] BUG: fix two bugs in yt.visualization.eps_writer where - a call to get_cmap was syntaxically erroneous - filename argument didn't accept Path objects --- nose_unit.cfg | 2 +- tests/tests.yaml | 1 + yt/visualization/eps_writer.py | 5 ++++- yt/visualization/tests/test_eps_writer.py | 27 +++++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 yt/visualization/tests/test_eps_writer.py diff --git a/nose_unit.cfg b/nose_unit.cfg index a34b709f157..67d7aba7110 100644 --- a/nose_unit.cfg +++ b/nose_unit.cfg @@ -6,5 +6,5 @@ nologcapture=1 verbosity=2 where=yt with-timer=1 -ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py) +ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py|test_eps_writer.py) exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF diff --git a/tests/tests.yaml b/tests/tests.yaml index 621bad06615..61be5178d54 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -186,6 +186,7 @@ other_tests: - '--ignore-files=test_load_sample.py' - '--ignore-files=test_field_access_pytest.py' - '--ignore-files=test_ambiguous_fields.py' + - '--ignore-files=test_eps_writer.py' - "--ignore-files=test_save.py" - '--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF' cookbook: diff --git a/yt/visualization/eps_writer.py b/yt/visualization/eps_writer.py index f489f969a5f..bcb9a1ad661 100644 --- a/yt/visualization/eps_writer.py +++ b/yt/visualization/eps_writer.py @@ -1,3 +1,5 @@ +import os + import numpy as np import pyx from matplotlib import cm, pyplot as plt @@ -743,7 +745,7 @@ def colorbar( # Convert the colormap into a string x = np.linspace(1, 0, 256) - cm_string = cm.get_cmap[name](x, bytes=True)[:, 0:3].tobytes() + cm_string = cm.get_cmap(name)(x, bytes=True)[:, 0:3].tobytes() cmap_im = pyx.bitmap.image(imsize[0], imsize[1], "RGB", cm_string) if orientation == "top" or orientation == "bottom": @@ -1143,6 +1145,7 @@ def save_fig(self, filename="test", format="eps", resolution=250): >>> d = DualEPS() >>> d.axis_box(xrange=(0, 100), yrange=(1e-3, 1), ylog=True) """ + filename = os.path.expanduser(filename) if format == "eps": self.canvas.writeEPSfile(filename) elif format == "pdf": diff --git a/yt/visualization/tests/test_eps_writer.py b/yt/visualization/tests/test_eps_writer.py new file mode 100644 index 00000000000..a047fe5e671 --- /dev/null +++ b/yt/visualization/tests/test_eps_writer.py @@ -0,0 +1,27 @@ +import yt +from yt.testing import fake_amr_ds, requires_module + + +@requires_module("pyx") +def test_eps_writer(tmp_path): + import yt.visualization.eps_writer as eps + + fields = [ + ("gas", "density"), + ("gas", "temperature"), + ] + units = [ + "g/cm**3", + "K", + ] + ds = fake_amr_ds(fields=fields, units=units) + slc = yt.SlicePlot( + ds, + "z", + fields=fields, + ) + eps_fig = eps.multiplot_yt(2, 1, slc, bare_axes=True) + eps_fig.scale_line(0.2, "5 cm") + savefile = tmp_path / "multi" + eps_fig.save_fig(savefile, format="eps") + assert savefile.with_suffix(".eps").exists() From 6c889fa80eabdc006d9c74ace9dee581523968c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Wed, 1 Sep 2021 14:17:38 +0200 Subject: [PATCH 59/60] ENH: skip eps_writer test if 'tex' executable isn't installed --- yt/testing.py | 24 +++++++++++++++++++++++ yt/visualization/tests/test_eps_writer.py | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/yt/testing.py b/yt/testing.py index 09832f9eca0..c021e8f047a 100644 --- a/yt/testing.py +++ b/yt/testing.py @@ -7,6 +7,7 @@ import shutil import tempfile import unittest +from shutil import which import matplotlib import numpy as np @@ -1364,6 +1365,29 @@ def ftrue(func): return ffalse +def requires_external_executable(*names): + import pytest + + def deco(func): + missing = [] + for name in names: + if which(name) is None: + missing.append(name) + + # note that order between these two decorators matters + @pytest.mark.skipif( + missing, + reason=f"missing external executable(s): {', '.join(missing)}", + ) + @functools.wraps(func) + def inner_func(*args, **kwargs): + return func(*args, **kwargs) + + return inner_func + + return deco + + class TempDirTest(unittest.TestCase): """ A test class that runs in a temporary directory and diff --git a/yt/visualization/tests/test_eps_writer.py b/yt/visualization/tests/test_eps_writer.py index a047fe5e671..5e8c6c1fe84 100644 --- a/yt/visualization/tests/test_eps_writer.py +++ b/yt/visualization/tests/test_eps_writer.py @@ -1,7 +1,8 @@ import yt -from yt.testing import fake_amr_ds, requires_module +from yt.testing import fake_amr_ds, requires_external_executable, requires_module +@requires_external_executable("tex") @requires_module("pyx") def test_eps_writer(tmp_path): import yt.visualization.eps_writer as eps From 84aa46b86d3d95fa3ad6a7dd0c2e274cd002df26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 6 Sep 2021 15:48:13 +0200 Subject: [PATCH 60/60] BUG: fix an outdated h5py idiom in a documentation notebook --- doc/source/examining/Loading_Generic_Array_Data.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/examining/Loading_Generic_Array_Data.ipynb b/doc/source/examining/Loading_Generic_Array_Data.ipynb index 6c99a10a48c..6af957a0949 100644 --- a/doc/source/examining/Loading_Generic_Array_Data.ipynb +++ b/doc/source/examining/Loading_Generic_Array_Data.ipynb @@ -266,7 +266,7 @@ }, "outputs": [], "source": [ - "data = {k:(v.value,u) for (k,v), u in zip(f.items(),units)}\n", + "data = {k:(v[()],u) for (k,v), u in zip(f.items(),units)}\n", "bbox = np.array([[-0.5, 0.5], [-0.5, 0.5], [-0.5, 0.5]])" ] },