From c8168f2e7fd1480a46a59c13afa5d2e67d428b71 Mon Sep 17 00:00:00 2001 From: John ZuHone Date: Sun, 9 Apr 2023 22:34:29 -0400 Subject: [PATCH 1/5] Export standard deviation to pandas DataFrames and AstroPy QTables --- yt/data_objects/profiles.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/yt/data_objects/profiles.py b/yt/data_objects/profiles.py index 1894876f872..aacd1556a84 100644 --- a/yt/data_objects/profiles.py +++ b/yt/data_objects/profiles.py @@ -562,7 +562,7 @@ def _export_prep(self, fields, only_used): fields = self.data_source._determine_fields(fields) return idxs, masked, fields - def to_dataframe(self, fields=None, only_used=False): + def to_dataframe(self, fields=None, only_used=False, include_std=False): r"""Export a profile object to a pandas DataFrame. This function will take a data object and construct from it and @@ -577,8 +577,13 @@ def to_dataframe(self, fields=None, only_used=False): profile, along with the bin field, will be exported. only_used : boolean, default False If True, only the bins which have data will be exported. If False, - all of the bins will be exported, but the elements for those bins + all the bins will be exported, but the elements for those bins in the data arrays will be filled with NaNs. + include_std : boolean, optional + If True, include the standard deviotion of the profile + in the pandas DataFrame. It will appear in the table as the + field name with "_stddev" appended, e.g. "velocity_x_stddev". + Default: False Returns ------- @@ -600,6 +605,8 @@ def to_dataframe(self, fields=None, only_used=False): pdata = {self.x_field[-1]: self.x[idxs]} for field in fields: pdata[field[-1]] = self[field][idxs] + if include_std: + pdata[f"{field[-1]}_stddev"] = self.standard_deviation[field][idxs] df = pd.DataFrame(pdata) if masked: mask = np.zeros(df.shape, dtype="bool") @@ -607,7 +614,7 @@ def to_dataframe(self, fields=None, only_used=False): df.mask(mask, inplace=True) return df - def to_astropy_table(self, fields=None, only_used=False): + def to_astropy_table(self, fields=None, only_used=False, include_std=False): """ Export the profile data to a :class:`~astropy.table.table.QTable`, which is a Table object which is unit-aware. The QTable can then @@ -627,10 +634,15 @@ def to_astropy_table(self, fields=None, only_used=False): to the QTable as rows. If False, all bins are copied, but the bins which are not used are masked. Default: False + include_std : boolean, optional + If True, include the standard deviotion of the profile + in the AstroPy QTable. It will appear in the table as the + field name with "_stddev" appended, e.g. "velocity_x_stddev". + Default: False Returns ------- - df : :class:`~astropy.table.QTable` + qt : :class:`~astropy.table.QTable` The data contained in the profile. Examples @@ -653,6 +665,12 @@ def to_astropy_table(self, fields=None, only_used=False): qt[field[-1]] = self[field][idxs].to_astropy() if masked: qt[field[-1]].mask = self.used + if include_std: + qt[f"{field[-1]}_stddev"] = self.standard_deviation[field][ + idxs + ].to_astropy() + if masked: + qt[f"{field[-1]}_stddev"].mask = self.used return qt From 4d458e4c10f70c8f532d94b346effd960ff63d69 Mon Sep 17 00:00:00 2001 From: John ZuHone Date: Mon, 10 Apr 2023 09:07:54 -0400 Subject: [PATCH 2/5] Docs --- doc/source/analyzing/generating_processed_data.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/source/analyzing/generating_processed_data.rst b/doc/source/analyzing/generating_processed_data.rst index 541a98a86b1..84abf87eb72 100644 --- a/doc/source/analyzing/generating_processed_data.rst +++ b/doc/source/analyzing/generating_processed_data.rst @@ -239,7 +239,9 @@ Exporting Profiles to DataFrame One-dimensional profile data can be exported to a :class:`~pandas.DataFrame` object using the :meth:`yt.data_objects.profiles.Profile1D.to_dataframe` method. Bins which do not have data will have their fields filled with ``NaN``, except for the bin field -itself. If you only want to export the bins which are used, set ``only_used=True``. +itself. If you only want to export the bins which are used, set ``only_used=True``, +and if you want to export the standard deviation of the profile as well, set +``include_std=True``: .. code-block:: python @@ -249,6 +251,8 @@ itself. If you only want to export the bins which are used, set ``only_used=True df_used = profile.to_dataframe(only_used=True) # Only adds the density and temperature fields df2 = profile.to_dataframe(fields=[("gas", "density"), ("gas", "temperature")]) + # Include standard deviation + df3 = profile.to_dataframe(include_std=True) The :class:`~pandas.DataFrame` can then analyzed and/or written to disk using pandas methods. Note that unit information is lost in this export. @@ -262,8 +266,9 @@ One-dimensional profile data also can be exported to an AstroPy :class:`~astropy object. This table can then be written to disk in a number of formats, such as ASCII text or FITS files, and manipulated in a number of ways. Bins which do not have data will have their mask values set to ``False``. If you only want to export the bins -which are used, set ``only_used=True``. Units are preserved in the table by converting -each :class:`~yt.units.yt_array.YTArray` to an :class:`~astropy.units.Quantity`. +which are used, set ``only_used=True``. If you want to include the standard deviation +of the field in the export, set ``include_std=True``. Units are preserved in the table +by converting each :class:`~yt.units.yt_array.YTArray` to an :class:`~astropy.units.Quantity`. To export the 1D profile to a Table object, simply call :meth:`yt.data_objects.profiles.Profile1D.to_astropy_table`: @@ -276,6 +281,8 @@ To export the 1D profile to a Table object, simply call t_used = profile.to_astropy_table(only_used=True) # Only adds the density and temperature fields t2 = profile.to_astropy_table(fields=[("gas", "density"), ("gas", "temperature")]) + # Export the standard deviation + t3 = profile.to_astropy_table(include_std=True) .. _generating-line-queries: From ff4a98d118e768e6a58977b46307be4e23dc0899 Mon Sep 17 00:00:00 2001 From: John ZuHone Date: Mon, 10 Apr 2023 09:15:17 -0400 Subject: [PATCH 3/5] Tests --- yt/data_objects/tests/test_profiles.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/yt/data_objects/tests/test_profiles.py b/yt/data_objects/tests/test_profiles.py index 7780fce7955..32d9f3cf17d 100644 --- a/yt/data_objects/tests/test_profiles.py +++ b/yt/data_objects/tests/test_profiles.py @@ -706,6 +706,15 @@ def test_export_astropy(): assert "velocity_x" not in at2.colnames assert_equal(prof.x.d[prof.used], at2["radius"].value) assert_equal(prof[("gas", "density")].d[prof.used], at2["density"].value) + at3 = prof.to_astropy_table(fields=("gas", "density"), include_std=True) + assert_equal(prof[("gas", "density")].d, at3["density"].value) + assert_equal( + prof.standard_deviation[("gas", "density")].d, at3["density_stddev"].value + ) + assert ( + prof.standard_deviation[("gas", "density")].units + == YTArray.from_astropy(at3["density_stddev"]).units + ) @requires_module("pandas") @@ -731,3 +740,8 @@ def test_export_pandas(): assert "velocity_x" not in df2.columns assert_equal(prof.x.d[prof.used], df2["radius"]) assert_equal(prof[("gas", "density")].d[prof.used], df2["density"]) + df3 = prof.to_dataframe(fields=("gas", "density"), include_std=True) + assert_equal( + prof.standard_deviation[("gas", "density")].d, + np.nan_to_num(df3["density_stddev"]), + ) From 5f394ef99c820cd08a28139fdeb2f4bf1275cbd3 Mon Sep 17 00:00:00 2001 From: John ZuHone Date: Wed, 12 Apr 2023 19:52:33 -0400 Subject: [PATCH 4/5] Update yt/data_objects/profiles.py Co-authored-by: Chris Havlin --- yt/data_objects/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/data_objects/profiles.py b/yt/data_objects/profiles.py index aacd1556a84..73bc4fb6966 100644 --- a/yt/data_objects/profiles.py +++ b/yt/data_objects/profiles.py @@ -580,7 +580,7 @@ def to_dataframe(self, fields=None, only_used=False, include_std=False): all the bins will be exported, but the elements for those bins in the data arrays will be filled with NaNs. include_std : boolean, optional - If True, include the standard deviotion of the profile + If True, include the standard deviation of the profile in the pandas DataFrame. It will appear in the table as the field name with "_stddev" appended, e.g. "velocity_x_stddev". Default: False From 5c50a7b8210f9163012e590c4347b691fb9e4f96 Mon Sep 17 00:00:00 2001 From: John ZuHone Date: Wed, 12 Apr 2023 19:52:41 -0400 Subject: [PATCH 5/5] Update yt/data_objects/profiles.py Co-authored-by: Chris Havlin --- yt/data_objects/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/data_objects/profiles.py b/yt/data_objects/profiles.py index 73bc4fb6966..933a669009b 100644 --- a/yt/data_objects/profiles.py +++ b/yt/data_objects/profiles.py @@ -635,7 +635,7 @@ def to_astropy_table(self, fields=None, only_used=False, include_std=False): copied, but the bins which are not used are masked. Default: False include_std : boolean, optional - If True, include the standard deviotion of the profile + If True, include the standard deviation of the profile in the AstroPy QTable. It will appear in the table as the field name with "_stddev" appended, e.g. "velocity_x_stddev". Default: False