Add C document to sphinx, fix arrow. (#8300)

- Group C API.
- Add C API sphinx doc.
- Consistent use of `OptionalArg` and the parameter name `config`.
- Remove call to deprecated functions in demo.
- Fix some formatting errors.
- Add links to c examples in the document (only visible with doxygen pages)
- Fix arrow.

.gitignore

@@ -52,6 +52,8 @@ Debug
 R-package.Rproj
 *.cache*
 .mypy_cache/
+doxygen
+
 # java
 java/xgboost4j/target
 java/xgboost4j/tmp

.readthedocs.yaml

@@ -5,13 +5,20 @@
 # Required
 version: 2
 
+submodules:
+  include: all
+
 # Set the version of Python and other tools you might need
 build:
   os: ubuntu-22.04
   tools:
     python: "3.8"
   apt_packages:
     - graphviz
+    - cmake
+    - g++
+    - doxygen
+    - ninja-build
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

demo/c-api/basic/c-api-demo.c

@@ -18,7 +18,7 @@ if (err != 0) {                                                         \
 }                                                                       \
 }
 
-int main(int argc, char** argv) {
+int main() {
   int silent = 0;
   int use_gpu = 0;  // set to 1 to use the GPU for training
 
@@ -67,10 +67,21 @@ int main(int argc, char** argv) {
 
   // predict
   bst_ulong out_len = 0;
-  const float* out_result = NULL;
   int n_print = 10;
 
-  safe_xgboost(XGBoosterPredict(booster, dtest, 0, 0, 0, &out_len, &out_result));
+  /* Run prediction with DMatrix object. */
+  char const config[] =
+      "{\"training\": false, \"type\": 0, "
+      "\"iteration_begin\": 0, \"iteration_end\": 0, \"strict_shape\": false}";
+  /* Shape of output prediction */
+  uint64_t const* out_shape;
+  /* Dimension of output prediction */
+  uint64_t out_dim;
+  /* Pointer to a thread local contigious array, assigned in prediction function. */
+  float const* out_result = NULL;
+  safe_xgboost(
+      XGBoosterPredictFromDMatrix(booster, dtest, config, &out_shape, &out_dim, &out_result));
+
   printf("y_pred: ");
   for (int i = 0; i < n_print; ++i) {
     printf("%1.4f ", out_result[i]);
@@ -98,12 +109,12 @@ int main(int argc, char** argv) {
     DMatrixHandle dmat;
     safe_xgboost(XGDMatrixCreateFromMat(values, 1, 127, 0.0, &dmat));
 
-    bst_ulong out_len = 0;
     const float* out_result = NULL;
 
-    safe_xgboost(XGBoosterPredict(booster, dmat, 0, 0, 0, &out_len,
-          &out_result));
-    assert(out_len == 1);
+    safe_xgboost(
+        XGBoosterPredictFromDMatrix(booster, dmat, config, &out_shape, &out_dim, &out_result));
+    assert(out_dim == 1);
+    assert(out_shape[0] == 1);
 
     printf("%1.4f \n", out_result[0]);
     safe_xgboost(XGDMatrixFree(dmat));
@@ -122,12 +133,12 @@ int main(int argc, char** argv) {
     safe_xgboost(XGDMatrixCreateFromCSREx(indptr, indices, data, 2, 22, 127,
       &dmat));
 
-    bst_ulong out_len = 0;
     const float* out_result = NULL;
 
-    safe_xgboost(XGBoosterPredict(booster, dmat, 0, 0, 0, &out_len,
-          &out_result));
-    assert(out_len == 1);
+    safe_xgboost(
+        XGBoosterPredictFromDMatrix(booster, dmat, config, &out_shape, &out_dim, &out_result));
+    assert(out_dim == 1);
+    assert(out_shape[0] == 1);
 
     printf("%1.4f \n", out_result[0]);
     safe_xgboost(XGDMatrixFree(dmat));
@@ -154,12 +165,12 @@ int main(int argc, char** argv) {
     safe_xgboost(XGDMatrixCreateFromCSCEx(col_ptr, indices, data, 128, 22, 1,
       &dmat));
 
-    bst_ulong out_len = 0;
     const float* out_result = NULL;
 
-    safe_xgboost(XGBoosterPredict(booster, dmat, 0, 0, 0, &out_len,
-          &out_result));
-    assert(out_len == 1);
+    safe_xgboost(
+        XGBoosterPredictFromDMatrix(booster, dmat, config, &out_shape, &out_dim, &out_result));
+    assert(out_dim == 1);
+    assert(out_shape[0] == 1);
 
     printf("%1.4f \n", out_result[0]);
     safe_xgboost(XGDMatrixFree(dmat));

demo/c-api/external-memory/external_memory.c

@@ -139,8 +139,8 @@ void TrainModel(DMatrix Xy) {
   Booster booster;
   DMatrix cache[] = {Xy};
   safe_xgboost(XGBoosterCreate(cache, 1, &booster));
-  /* Use approx for external memory training. */
-  safe_xgboost(XGBoosterSetParam(booster, "tree_method", "approx"));
+  /* Use approx or hist for external memory training. */
+  safe_xgboost(XGBoosterSetParam(booster, "tree_method", "hist"));
   safe_xgboost(XGBoosterSetParam(booster, "objective", "reg:squarederror"));
 
   /* Start training. */

doc/Doxyfile.in

@@ -753,7 +753,7 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = @PROJECT_SOURCE_DIR@/include @PROJECT_SOURCE_DIR@/src/common
+INPUT                  = @PROJECT_SOURCE_DIR@/include
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -822,7 +822,7 @@ EXCLUDE_SYMBOLS        =
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           =
+EXAMPLE_PATH           = @PROJECT_SOURCE_DIR@/demo/c-api/
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
@@ -836,7 +836,7 @@ EXAMPLE_PATTERNS       =
 # irrespective of the value of the RECURSIVE tag.
 # The default value is: NO.
 
-EXAMPLE_RECURSIVE      = NO
+EXAMPLE_RECURSIVE      = YES
 
 # The IMAGE_PATH tag can be used to specify one or more files or directories
 # that contain images that are to be included in the documentation (see the
@@ -1934,15 +1934,15 @@ ENABLE_PREPROCESSING   = YES
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
 # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
 # the macro expansion is limited to the macros specified with the PREDEFINED and
 # EXPAND_AS_DEFINED tags.
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_ONLY_PREDEF     = NO
+EXPAND_ONLY_PREDEF     = YES
 
 # If the SEARCH_INCLUDES tag is set to YES the includes files in the
 # INCLUDE_PATH will be searched if a #include is found.
@@ -1974,7 +1974,9 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             = DMLC_USE_CXX11
+PREDEFINED             = DMLC_USE_CXX11 \
+                         "XGB_DLL=" \
+                         "XGB_EXTERN_C="
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The

doc/c.rst

@@ -6,7 +6,59 @@ XGBoost implements a set of C API designed for various bindings, we maintain its
 and the CMake/make build interface.  See :doc:`/tutorials/c_api_tutorial` for an
 introduction and ``demo/c-api/`` for related examples.  Also one can generate doxygen
 document by providing ``-DBUILD_C_DOC=ON`` as parameter to ``CMake`` during build, or
-simply look at function comments in ``include/xgboost/c_api.h``.
+simply look at function comments in ``include/xgboost/c_api.h``. The reference is exported
+to sphinx with the help of breathe, which doesn't contain links to examples but might be
+easier to read. For the original doxygen pages please visit:
 
 * `C API documentation (latest master branch) <https://xgboost.readthedocs.io/en/latest/dev/c__api_8h.html>`_
 * `C API documentation (last stable release) <https://xgboost.readthedocs.io/en/stable/dev/c__api_8h.html>`_
+
+***************
+C API Reference
+***************
+
+.. contents::
+  :backlinks: none
+  :local:
+
+Library
+=======
+
+.. doxygengroup:: Library
+   :project: xgboost
+
+DMatrix
+=======
+
+.. doxygengroup:: DMatrix
+   :project: xgboost
+
+Streaming
+---------
+
+.. doxygengroup:: Streaming
+   :project: xgboost
+
+Booster
+=======
+
+.. doxygengroup:: Booster
+   :project: xgboost
+
+Prediction
+----------
+
+.. doxygengroup:: Prediction
+   :project: xgboost
+
+Serialization
+-------------
+
+.. doxygengroup:: Serialization
+   :project: xgboost
+
+Collective
+==========
+
+.. doxygengroup:: Collective
+   :project: xgboost

doc/conf.py

@@ -57,22 +57,24 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
-libpath = os.path.join(curr_path, '../python-package/')
+CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+PROJECT_ROOT = os.path.normpath(os.path.join(CURR_PATH, os.path.pardir))
+libpath = os.path.join(PROJECT_ROOT, "python-package/")
 sys.path.insert(0, libpath)
-sys.path.insert(0, curr_path)
+sys.path.insert(0, CURR_PATH)
 
 # -- General configuration ------------------------------------------------
 
 # General information about the project.
-project = u'xgboost'
-author = u'%s developers' % project
-copyright = u'2021, %s' % author
-github_doc_root = 'https://github.com/dmlc/xgboost/tree/master/doc/'
+project = "xgboost"
+author = "%s developers" % project
+copyright = "2022, %s" % author
+github_doc_root = "https://github.com/dmlc/xgboost/tree/master/doc/"
 
-os.environ['XGBOOST_BUILD_DOC'] = '1'
+os.environ["XGBOOST_BUILD_DOC"] = "1"
 # Version information.
-import xgboost                  # NOQA
+import xgboost  # NOQA
+
 version = xgboost.__version__
 release = xgboost.__version__
 
@@ -105,7 +107,10 @@
 plot_html_show_formats = False
 
 # Breathe extension variables
-breathe_projects = {"xgboost": "doxyxml/"}
+DOX_DIR = "doxygen"
+breathe_projects = {
+    "xgboost": os.path.join(PROJECT_ROOT, DOX_DIR, "doc_doxygen/xml")
+}
 breathe_default_project = "xgboost"
 
 # Add any paths that contain templates here, relative to this directory.
@@ -216,23 +221,29 @@
 
 
 # hook for doxygen
-def run_doxygen(folder):
+def run_doxygen():
     """Run the doxygen make command in the designated folder."""
+    curdir = os.path.normpath(os.path.abspath(os.path.curdir))
     try:
-        retcode = subprocess.call("cd %s; make doxygen" % folder, shell=True)
-        if retcode < 0:
-            sys.stderr.write("doxygen terminated by signal %s" % (-retcode))
+        os.chdir(PROJECT_ROOT)
+        if not os.path.exists(DOX_DIR):
+            os.mkdir(DOX_DIR)
+        os.chdir(os.path.join(PROJECT_ROOT, DOX_DIR))
+        subprocess.check_call(["cmake", "..", "-DBUILD_C_DOC=ON", "-GNinja"])
+        subprocess.check_call(["ninja", "doc_doxygen"])
     except OSError as e:
         sys.stderr.write("doxygen execution failed: %s" % e)
+    finally:
+        os.chdir(curdir)
 
 
 def generate_doxygen_xml(app):
     """Run the doxygen make commands if we're on the ReadTheDocs server"""
     read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'
     if read_the_docs_build:
-        run_doxygen('..')
+        run_doxygen()
 
 
-# app.add_stylesheet() is deprecated. Use app.add_css_file()
 def setup(app):
     app.add_css_file('custom.css')
+    app.connect("builder-inited", generate_doxygen_xml)

doc/contrib/docs.rst

@@ -11,7 +11,7 @@ Documentation and Examples
 *********
 Documents
 *********
-* Documentation is built using `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
+* Python and C documentation is built using `Sphinx <http://www.sphinx-doc.org/en/master/>`_.
 * Each document is written in `reStructuredText <http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`_.
 * You can build document locally to see the effect, by running
 

doc/tutorials/c_api_tutorial.rst

@@ -2,7 +2,7 @@
 C API Tutorial
 ##############
 
-In this tutorial, we are going to install XGBoost library & configure the CMakeLists.txt file of our C/C++ application to link XGBoost library with our application. Later on, we will see some useful tips for using C API and code snippets as examples to use various functions available in C API to perform basic task like loading, training model & predicting on test dataset.
+In this tutorial, we are going to install XGBoost library & configure the CMakeLists.txt file of our C/C++ application to link XGBoost library with our application. Later on, we will see some useful tips for using C API and code snippets as examples to use various functions available in C API to perform basic task like loading, training model & predicting on test dataset. For API reference, please visit :doc:`/c`
 
 .. contents::
   :backlinks: none