Merge pull request #2437 from matthewturk/yt4-merge

merging yt-4 into master

.gitignore

@@ -21,6 +21,7 @@ yt/geometry/oct_container.c
 yt/geometry/oct_visitors.c
 yt/geometry/particle_deposit.c
 yt/geometry/particle_oct_container.c
+yt/geometry/particle_oct_container.cpp
 yt/geometry/particle_smooth.c
 yt/geometry/selection_routines.c
 yt/utilities/amr_utils.c
@@ -34,11 +35,16 @@ yt/utilities/lib/alt_ray_tracers.c
 yt/utilities/lib/amr_kdtools.c
 yt/utilities/lib/basic_octree.c
 yt/utilities/lib/bitarray.c
+yt/utilities/lib/bounded_priority_queue.c
 yt/utilities/lib/bounding_volume_hierarchy.c
 yt/utilities/lib/contour_finding.c
+yt/utilities/lib/cykdtree/kdtree.cpp
+yt/utilities/lib/cykdtree/utils.cpp
+yt/utilities/lib/cyoctree.cpp
 yt/utilities/lib/depth_first_octree.c
 yt/utilities/lib/distance_queue.c
 yt/utilities/lib/element_mappings.c
+yt/utilities/lib/ewah_bool_wrap.cpp
 yt/utilities/lib/fnv_hash.c
 yt/utilities/lib/fortran_reader.c
 yt/utilities/lib/freetype_writer.c
@@ -55,13 +61,16 @@ yt/utilities/lib/mesh_samplers.cpp
 yt/utilities/lib/mesh_traversal.cpp
 yt/utilities/lib/mesh_triangulation.c
 yt/utilities/lib/mesh_utilities.c
+yt/utilities/lib/pixelization_routines.cpp
 yt/utilities/lib/misc_utilities.c
+yt/utilities/lib/particle_kdtree_tools.cpp
 yt/utilities/lib/particle_mesh_operations.c
 yt/utilities/lib/partitioned_grid.c
 yt/utilities/lib/primitives.c
 yt/utilities/lib/origami.c
 yt/utilities/lib/particle_mesh_operations.c
 yt/utilities/lib/pixelization_routines.c
+yt/utilities/lib/pixelization_routines.cpp
 yt/utilities/lib/png_writer.c
 yt/utilities/lib/points_in_volume.c
 yt/utilities/lib/quad_tree.c
@@ -71,7 +80,6 @@ yt/utilities/lib/grid_traversal.c
 yt/utilities/lib/marching_cubes.c
 yt/utilities/lib/png_writer.h
 yt/utilities/lib/write_array.c
-yt/utilities/lib/perftools_wrap.c
 yt/utilities/lib/partitioned_grid.c
 yt/utilities/lib/volume_container.c
 yt/utilities/lib/lenses.c

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "answer-store"]
 	path = answer-store
 	url = https://github.com/yt-project/answer-store
-	branch = master
+	branch = yt-4.0

.hgignore

@@ -37,6 +37,7 @@ yt/utilities/lib/contour_finding.c
 yt/utilities/lib/depth_first_octree.c
 yt/utilities/lib/distance_queue.c
 yt/utilities/lib/element_mappings.c
+yt/utilities/lib/ewah_bool_wrap.cpp
 yt/utilities/lib/fnv_hash.c
 yt/utilities/lib/fortran_reader.c
 yt/utilities/lib/freetype_writer.c
@@ -54,6 +55,7 @@ yt/utilities/lib/mesh_traversal.cpp
 yt/utilities/lib/mesh_triangulation.c
 yt/utilities/lib/mesh_utilities.c
 yt/utilities/lib/misc_utilities.c
+yt/utilities/lib/particle_kdtree_tools.cpp
 yt/utilities/lib/particle_mesh_operations.c
 yt/utilities/lib/partitioned_grid.c
 yt/utilities/lib/primitives.c

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-dist: xenial
+dist: bionic
 cache:
   pip: true
   directories:
@@ -9,6 +9,7 @@ addons:
   apt:
     packages:
       - libhdf5-serial-dev
+      - libnetcdf-dev
       - libproj-dev
       - proj-data
       - proj-bin
@@ -55,9 +56,11 @@ install:
     fi
     if [[ ${TRAVIS_BUILD_STAGE_NAME} != "Lint" ]]; then
       if [[ $MINIMAL == 1 ]]; then
+        # Ensure numpy and cython are installed so dependencies that need to be built
+        # don't error out
         # The first numpy to support py3.6 is 1.12, but numpy 1.13 matches 
         # unyt so we'll match it here. 
-        $PIP install numpy==1.13.3 cython==0.24
+        $PIP install numpy==1.13.3 cython==0.26.1
         $PIP install -r tests/test_minimal_requirements.txt
       else
         # Getting cartopy installed requires getting cython and numpy installed
@@ -95,9 +98,12 @@ jobs:
       python: 3.8
       script: coverage run $(which nosetests) -c nose_unit.cfg
 
+      # This is not necessarily going to be forever -- once we merge yt-4.0
+      # with master we will likely change this around to reduce the number of
+      # versions we test on.
     - stage: tests
-      name: "Python: 3.6 Answer Tests"
-      python: 3.6
+      name: "Python: 3.7 Answer Tests"
+      python: 3.7
       script: coverage run $(which nosetests) -c nose_answer.cfg
       after_failure: python tests/report_failed_answers.py -f -m --xunit-file "answer_nosetests.xml"
 

CONTRIBUTING.rst

@@ -16,12 +16,8 @@ Coding is only one way to be involved!
 Communication Channels
 ----------------------
 
-There are five main communication channels for yt:
+There are three main communication channels for yt:
 
- * We have an IRC channel, on ``irc.freenode.net`` in ``#yt``.
-   You can connect through our web
-   gateway without any special client, at https://yt-project.org/irc.html .
-   *IRC is the first stop for conversation!*
  * Many yt developers participate in the yt Slack community. Slack is a free
    chat service that many teams use to organize their work. You can get an
    invite to yt's Slack organization by clicking the "Join us @ Slack" button
@@ -405,12 +401,6 @@ the following subdirectories:
    classes for data regions, covering grids, time series, and so on.  This
    also includes derived fields and derived quantities.
 
-``analysis_modules``
-   This is where all mechanisms for processing data live.  This includes
-   things like clump finding, halo profiling, halo finding, and so on.  This
-   is something of a catchall, but it serves as a level of greater
-   abstraction that simply data selection and modification.
-
 ``gui``
    This is where all GUI components go.  Typically this will be some small
    tool used for one or two things, which contains a launching mechanism on
@@ -762,6 +752,7 @@ Source code style guide
 
  * In general, follow PEP-8 guidelines.
    https://www.python.org/dev/peps/pep-0008/
+ * We no longer have a copyright blurb in every source file.
  * Classes are ``ConjoinedCapitals``, methods and functions are
    ``lowercase_with_underscores``.
  * Use 4 spaces, not tabs, to represent indentation.
@@ -784,7 +775,7 @@ Source code style guide
    that occur on an object.  See :ref:`docstrings` below for a fiducial example
    of a docstring.
  * Use only one top-level import per line. Unless there is a good reason not to,
-   imports should happen at the top of the file, after the copyright blurb.
+   imports should happen at the top of the file.
  * Never compare with ``True`` or ``False`` using ``==`` or ``!=``, always use
    ``is`` or ``is not``.
  * If you are comparing with a numpy boolean array, just refer to the array.

MANIFEST.in

@@ -3,14 +3,14 @@ include yt/visualization/mapserver/html/map.js
 include yt/visualization/mapserver/html/map_index.html
 include yt/utilities/mesh_types.yaml
 exclude scripts/pr_backport.py
-recursive-include yt *.py *.pyx *.pxd *.h README* *.txt LICENSE* *.cu
+recursive-include yt *.py *.pyx *.pxd *.h *.hpp README* *.txt LICENSE* *.cu
 recursive-include doc *.rst *.txt *.py *.ipynb *.png *.jpg *.css *.html
 recursive-include doc *.h *.c *.sh *.svgz *.pdf *.svg *.pyx
 include doc/README doc/activate doc/activate.csh doc/cheatsheet.tex
 include doc/extensions/README doc/Makefile
 prune doc/source/reference/api/generated
 prune doc/build
-recursive-include yt/analysis_modules/halo_finding/rockstar *.py *.pyx
 recursive-include yt/visualization/volume_rendering/shaders *.fragmentshader *.vertexshader
+include yt/sample_data_registry.json
 prune yt/frontends/_skeleton
 recursive-include yt/frontends/amrvac *.par

appveyor.yml

@@ -30,7 +30,10 @@ install:
     - "python --version"
 
     # Install specified version of numpy and dependencies
-    - "conda install --yes -c conda-forge numpy scipy nose pytest setuptools ipython Cython sympy fastcache h5py matplotlib=3.1.3 mock pandas cartopy conda-build pyyaml"
+    - "conda install --yes -c conda-forge numpy scipy nose pytest setuptools ipython git
+    Cython sympy fastcache h5py matplotlib=3.1.3 mock pandas cartopy conda-build pooch pyyaml"
+    - "pip install git+https://github.com/yt-project/unyt@de443dff7671f1e68557306d77582cd117cc94f8#egg=unyt"
+    # install yt
     - "pip install -e ."
 
 # Not a .NET project

doc/Makefile

@@ -49,8 +49,7 @@ html:
 ifneq ($(READTHEDOCS),True)
 	SPHINX_APIDOC_OPTIONS=members,undoc-members,inherited-members,show-inheritance sphinx-apidoc \
         -o source/reference/api/ \
-        -e ../yt ../yt/extern* $(shell find ../yt -name "*tests*" -type d) ../yt/utilities/voropp* \
-           ../yt/analysis_modules/halo_finding/{fof,hop}
+        -e ../yt ../yt/extern* $(shell find ../yt -name "*tests*" -type d) ../yt/utilities/voropp*
 endif
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo

doc/helper_scripts/parse_cb_list.py

@@ -28,7 +28,7 @@ def write_docstring(f, name, cls):
     sig = sig.replace("**kwargs", "**field_parameters")
     clsproxy = "yt.visualization.plot_modifications.%s" % (cls.__name__)
     #docstring = "\n".join(["   %s" % line for line in docstring.split("\n")])
-    #print docstring
+    #print(docstring)
     f.write(template % dict(clsname = clsname, sig = sig, clsproxy=clsproxy,
                             docstring = "\n".join(tw.wrap(docstring))))
                             #docstring = docstring))

doc/helper_scripts/show_fields.py

@@ -54,8 +54,10 @@ def _strip_ftype(field):
                          unit_registry=ds.unit_registry)
 for my_unit in ["m", "pc", "AU", "au"]:
     new_unit = "%scm" % my_unit
-    ds.unit_registry.add(new_unit, base_ds.unit_registry.lut[my_unit][0],
-                         dimensions.length, "\\rm{%s}/(1+z)" % my_unit)
+    my_u = Unit(my_unit, registry=ds.unit_registry)
+    ds.unit_registry.add(new_unit, my_u.base_value,
+                         dimensions.length, "\\rm{%s}/(1+z)" % my_unit,
+                         prefixable=True)
 
 
 
@@ -143,7 +145,7 @@ def print_all_fields(fl):
                 print("   * Units: :math:`%s`" % fix_units(df.units))
             else:
                 print("   * Units: :math:`%s`" % fix_units(df.units, in_cgs=True))
-        print("   * Particle Type: %s" % (df.particle_type))
+        print("   * Sampling Method: %s" % (df.sampling_type))
         print()
         print("**Field Source**")
         print()

doc/helper_scripts/table.py

@@ -23,13 +23,6 @@
      ("interacting/index.html", "Interacting with yt",
       "Different ways -- scripting, GUIs, prompts, explorers -- to explore " +
       "your data."),
-     ("analysis_modules/index.html", "Analysis Modules",
-      "Discussions of some provided procedures for astrophysical analysis " +
-      "like halo finding and synthetic spectra.  Halo finding, analyzing " +
-      "cosmology simulations, halo mass functions, halo profiling, light " +
-      "cone generator, making absorption spectrums, star particle " +
-      "analysis, two-point functions, halo merger trees, clump finding, " +
-      "radial column density, exporting to sunrise.")
   ]),
   ("Advanced Usage", [
      ("advanced/index.html", "Advanced yt usage",

doc/install_script.sh

@@ -32,13 +32,13 @@ INST_GIT=1      # Install git or not?  If git is not already installed, yt
 INST_EMBREE=0   # Install dependencies needed for Embree-accelerated ray tracing
 INST_PYX=0      # Install PyX?  Sometimes PyX can be problematic without a
                 # working TeX installation.
-INST_ROCKSTAR=0 # Install the Rockstar halo finder?
 INST_SCIPY=0    # Install scipy?
 INST_H5PY=1     # Install h5py?
 INST_ASTROPY=0  # Install astropy?
 INST_CARTOPY=0  # Install cartopy?
 INST_NOSE=1     # Install nose?
 INST_NETCDF4=1  # Install netcdf4 and its python bindings?
+INST_POOCH=1    # Install pooch?
 INST_HG=0       # Install Mercurial or not?
 
 # This is the branch we will install from for INST_YT_SOURCE=1
@@ -134,13 +134,13 @@ function write_config
     echo INST_GIT=${INST_GIT} >> ${CONFIG_FILE}
     echo INST_PYX=${INST_PYX} >> ${CONFIG_FILE}
     echo INST_PY3=${INST_PY3} >> ${CONFIG_FILE}
-    echo INST_ROCKSTAR=${INST_ROCKSTAR} >> ${CONFIG_FILE}
     echo INST_SCIPY=${INST_SCIPY} >> ${CONFIG_FILE}
     echo INST_EMBREE=${INST_EMBREE} >> ${CONFIG_FILE}
     echo INST_H5PY=${INST_H5PY} >> ${CONFIG_FILE}
     echo INST_ASTROPY=${INST_ASTROPY} >> ${CONFIG_FILE}
     echo INST_CARTOPY=${INST_CARTOPY} >> ${CONFIG_FILE}
     echo INST_NOSE=${INST_NOSE} >> ${CONFIG_FILE}
+    echo INST_POOCH=${INST_POOCH} >> ${CONFIG_FILE}
 
     echo YT_DIR=${YT_DIR} >> ${CONFIG_FILE}
 }
@@ -286,17 +286,6 @@ then
     PYEMBREE_URL="https://github.com/scopatz/pyembree/archive/master.zip"
 fi
 
-if [ $INST_ROCKSTAR -ne 0 ]
-then
-    if [ $INST_YT_SOURCE -eq 0 ]
-    then
-        echo "yt must be compiled from source to install support for"
-        echo "the rockstar halo finder. Please set INST_YT_SOURCE to 1"
-        echo "and re-run the install script"
-        exit 1
-    fi
-fi
-
 echo
 echo
 echo "========================================================================"
@@ -330,10 +319,6 @@ printf "%-18s = %s so I " "INST_PYX" "${INST_PYX}"
 get_willwont ${INST_PYX}
 echo "be installing PyX"
 
-printf "%-18s = %s so I " "INST_ROCKSTAR" "${INST_ROCKSTAR}"
-get_willwont ${INST_ROCKSTAR}
-echo "be installing Rockstar"
-
 printf "%-18s = %s so I " "INST_H5PY" "${INST_H5PY}"
 get_willwont ${INST_H5PY}
 echo "be installing h5py"
@@ -350,6 +335,10 @@ printf "%-18s = %s so I " "INST_NOSE" "${INST_NOSE}"
 get_willwont ${INST_NOSE}
 echo "be installing nose"
 
+printf "%-18s = %s so I " "INST_POOCH" "${INST_POOCH}"
+get_willwont ${INST_POOCH}
+echo "be installing pooch"
+
 echo
 
 echo
@@ -522,6 +511,10 @@ if [ $INST_CARTOPY -ne 0 ]
 then
     YT_DEPS+=('cartopy')
 fi
+if [ $INST_POOCH -ne 0 ]
+then
+    YT_DEPS+=('pooch')
+fi
 YT_DEPS+=('conda-build')
 if [ $INST_PY3 -eq 0 ] && [ $INST_HG -eq 1 ]
 then
@@ -598,16 +591,6 @@ then
     popd &> /dev/null
 fi
 
-if [ $INST_ROCKSTAR -eq 1 ]
-then
-    echo "Building Rockstar"
-    ( ${GIT_EXE} clone https://github.com/yt-project/rockstar ${DEST_DIR}/src/rockstar/ 2>&1 ) 1>> ${LOG_FILE}
-    ROCKSTAR_PACKAGE=$(${DEST_DIR}/bin/conda build ${DEST_DIR}/src/yt_conda/rockstar --output)
-    log_cmd ${DEST_DIR}/bin/conda build ${DEST_DIR}/src/yt_conda/rockstar
-    log_cmd ${DEST_DIR}/bin/conda install $ROCKSTAR_PACKAGE
-    ROCKSTAR_DIR=${DEST_DIR}/src/rockstar
-fi
-
 # conda doesn't package pyx, so we install manually with pip
 if [ $INST_PYX -eq 1 ]
 then
@@ -650,13 +633,8 @@ else
     then
         echo $DEST_DIR > ${YT_DIR}/embree.cfg
     fi
-    if [ $INST_ROCKSTAR -eq 1 ]
-    then
-        echo $ROCKSTAR_DIR > ${YT_DIR}/rockstar.cfg
-        ROCKSTAR_LIBRARY_PATH=${DEST_DIR}/lib
-    fi
     pushd ${YT_DIR} &> /dev/null
-    ( LIBRARY_PATH=$ROCKSTAR_LIBRARY_PATH ${DEST_DIR}/bin/${PYTHON_EXEC} setup.py develop 2>&1) 1>> ${LOG_FILE} || do_exit
+    ( ${DEST_DIR}/bin/${PYTHON_EXEC} setup.py develop 2>&1) 1>> ${LOG_FILE} || do_exit
     popd &> /dev/null
 fi
 
@@ -697,21 +675,6 @@ echo "You can also update the init file appropriate for your shell"
 echo "(e.g. .bashrc, .bash_profile, .cshrc, or .zshrc) to include"
 echo "the same command."
 echo
-if [ $INST_ROCKSTAR -eq 1 ]
-then
-    if [ $MYOS = "Darwin" ]
-    then
-        LD_NAME="DYLD_LIBRARY_PATH"
-    else
-        LD_NAME="LD_LIBRARY_PATH"
-    fi
-    echo
-    echo "For rockstar to work, you must also set $LD_NAME:"
-    echo
-    echo "    export $LD_NAME=$DEST_DIR/lib:\$$LD_NAME"
-    echo
-    echo "or whichever invocation is appropriate for your shell."
-fi
 echo "========================================================================"
 echo
 echo "Oh, look at me, still talking when there's science to do!"