Add support for foreign libraries (#2659)

RFC: #2650

Signed-off-by: Andrey Mokhov <amokhov@janestreet.com>

CHANGES.md

@@ -102,6 +102,8 @@
 
 - `c_flags`, `c_names` and `cxx_names` are now supported in `executable`
   and `executables` stanzas. (#2562, @nojb)
+  Note: this feature has been subsequently extended into a separate
+  `foreign_stubs` field. (#2659, RFC #2650, @snowleopard)
 
 - Remove git integration from `$ dune upgrade` (#2565, @rgrinberg)
 
@@ -158,6 +160,11 @@
   projects where the language is at least `2.0`, the field is now forbidden.
   (#2752, fixes #2747, @rgrinberg)
 
+- Extend support for foreign sources and archives via the `(foreign_library ...)`
+  stanza as well as the `(foreign_stubs ...)` and `(foreign_archives ...)` fields.
+  (#2659, RFC #2650, @snowleopard)
+
+
 1.11.4 (09/10/2019)
 -------------------
 

bin/arg.ml

@@ -23,7 +23,7 @@ let profile =
     , fun fmt t -> Format.pp_print_string fmt (Profile.to_string t) )
 
 module Dep = struct
-  module Dep_conf = Dune_file.Dep_conf
+  module Dep_conf = Dep_conf
 
   type t = Dep_conf.t
 

bin/arg.mli

@@ -14,7 +14,7 @@ module Path : sig
 end
 
 module Dep : sig
-  type t = Dune_file.Dep_conf.t
+  type t = Dep_conf.t
 
   val file : string -> t
 

bin/target.ml

@@ -132,7 +132,7 @@ let resolve_alias common ~recursive sv ~(setup : Dune.Main.build_system) =
   | None -> Error [ Pp.text "alias cannot contain variables" ]
 
 let resolve_target common ~setup = function
-  | Dune.Dune_file.Dep_conf.Alias sv as dep ->
+  | Dune.Dep_conf.Alias sv as dep ->
     Result.map_error
       ~f:(fun hints -> (dep, hints))
       (resolve_alias common ~recursive:false sv ~setup)

doc/concepts.rst

@@ -1057,3 +1057,109 @@ Other files
 
 For all other kinds of elements, you need to attach them manually via
 an :ref:`install` stanza.
+
+
+.. _foreign-sources-and-archives:
+
+Foreign sources and archives
+============================
+
+Dune provides basic support for including foreign source files as well
+as archives of foreign object files into OCaml projects via the
+``foreign_stubs`` and ``foreign_archives`` fields.
+
+.. _foreign-stubs:
+
+Foreign stubs
+-------------
+
+You can specify foreign sources using the ``foreign_stubs`` field of the
+``library`` and ``executable`` stanzas. For example:
+
+.. code:: scheme
+
+    (library
+     (name lib)
+     (foreign_stubs (language c) (names src1 src2))
+     (foreign_stubs (language cxx) (names src3) (flags -O2)))
+
+Here we declare an OCaml library ``lib``, which contains two C sources
+``src1`` and ``src2``, and one C++ source ``src3`` that needs to be
+compiled with ``-O2``. These source files will be compiled and packaged
+with the library, along with the link-time flags to be used when
+linking the final executables. When matching ``names`` to source files,
+Dune treats ``*.c`` files as C sources, and ``*.cpp``, ``*.cc`` and
+``*.cxx`` files as C++ sources.
+
+Here is a complete list of supported subfields:
+
+- ``language`` specifies the source language, where ``c`` means C and
+  ``cxx`` means C++. In future, more languages may be supported.
+- ``names`` specifies the *names* of source files. When specifying a source
+  file, you should omit the extension and any relative parts of the path;
+  Dune will scan all library directories, finding all matching files and
+  raising an error if multiple source files map to the same object name.
+  If you need to have multiple object files with the same name, you can
+  package them into different :ref:`foreign-archives` via the
+  ``foreign_archives`` field.
+- ``flags`` are passed when compiling source files. This field is specified
+  using the :ref:`ordered-set-language`, where the ``:standard`` value comes
+  from the environment settings ``c_flags`` and ``cxx_flags``, respectively.
+- ``include_dirs`` are tracked as dependencies and passed to the compiler
+  via the ``-I`` flag. You can use :ref:`variables` in this field. The
+  contents of included directories is tracked recursively, e.g. if you
+  use ``(include_dir dir)`` and have headers ``dir/base.h`` and
+  ``dir/lib/lib.h`` then they both will be tracked as dependencies.
+- ``extra_deps`` specifies any other dependencies that should be tracked.
+  This is useful when dealing with ``#include`` statements that escape into
+  a parent directory like ``#include "../a.h"``.
+
+
+.. _foreign-archives:
+
+Foreign archives
+----------------
+
+You can also specify archives of separately compiled foreign object files
+that need to be packaged with an OCaml library or linked into an OCaml
+executable. To do that, use the ``foreign_archives`` field of the
+corresponding ``library`` or ``executable`` stanza. For example:
+
+.. code:: scheme
+
+    (library
+     (name lib)
+     (foreign_stubs (language c) (names src1 src2))
+     (foreign_stubs (language cxx) (names src3) (flags -O2))
+     (foreign_archives arch1 some/dir/arch2))
+
+Here, in addition to :ref:`foreign-stubs`, we also specify foreign archives
+``arch1`` and ``arch2``, where the latter is stored in a subdirectory
+``some/dir``.
+
+You can build a foreign archive manually, e.g. using a custom ``rule`` as
+described in :ref:`foreign-sandboxing`, or ask Dune to build it via the
+``foreign_library`` stanza:
+
+.. code:: scheme
+
+    (foreign_library
+     (archive_name arch1)
+     (language c)
+     (names src4 src5)
+     (include_dir headers))
+
+This asks Dune to compile C source files ``src4`` and ``src5`` with
+headers tracked in the ``headers`` directory, and put the resulting
+object files into an archive ``arch1``, whose full name is typically
+``libarch1.a`` for static linking and ``dllarch1.so`` for dynamic
+linking.
+
+The ``foreign_library`` stanza supports all :ref:`foreign-stubs` fields plus
+the ``archive_name`` field, which specifies the archive's name. You can refer
+to the same archive name from multiple OCaml libraries and executables, so a
+foreign archive is a bit like a foreign library, hence the name of the stanza.
+
+Foreign archives are particularly useful when embedding a library written in
+a foreign language and/or built with another build system. See
+:ref:`foreign-sandboxing` for more details.

doc/dune-files.rst

@@ -372,10 +372,16 @@ to use the :ref:`include_subdirs` stanza.
   or in the installed world. You can use this to provide extra features without
   adding hard dependencies to your project
 
-- ``(c_names (<names>))``, if your library has stubs, you must list the C files
-  in this field, without the ``.c`` extension
+- ``(foreign_stubs <foreign-stubs-spec>)`` specifies foreign source files, e.g.
+  C or C++ stubs, to be compiled and packaged together with the library. See
+  the section :ref:`foreign-sources-and-archives` for more details. This field
+  replaces the now deprecated fields ``c_names``, ``c_flags``, ``cxx_names``
+  and ``cxx_flags``.
 
-- ``(cxx_names (<names>))`` is the same as ``c_names`` but for C++ stubs
+- ``(foreign_archives <foreign-archives-list>)`` specifies archives of foreign
+  object files to be packaged with the library. See the section
+  :ref:`foreign-archives` for more details. This field replaces the now
+  deprecated field ``self_build_stubs_archive``.
 
 - ``(install_c_headers (<names>))``, if your library has public C header files
   that must be installed, you must list them in this field, without the ``.h``
@@ -422,26 +428,13 @@ to use the :ref:`include_subdirs` stanza.
   use this to specify ``-linkall`` for instance. ``<flags>`` is a list of
   strings supporting :ref:`variables`
 
-- ``(c_flags <flags>)`` specifies the compilation flags for C stubs, using the
-  :ref:`ordered-set-language`. This field supports ``(:include ...)`` forms
-
-- ``(cxx_flags <flags>)`` is the same as ``c_flags`` but for C++ stubs
-
 - ``(c_library_flags <flags>)`` specifies the flags to pass to the C compiler
   when constructing the library archive file for the C stubs. ``<flags>`` uses
   the :ref:`ordered-set-language` and supports ``(:include ...)`` forms. When you
   are writing bindings for a C library named ``bar``, you should typically write
   ``-lbar`` here, or whatever flags are necessary to to link against this
   library
 
-.. _self_build_stubs_archive:
-
-- ``(self_build_stubs_archive <c-libname>)`` indicates to dune that the
-  library has stubs, but that the stubs are built manually. The aim of the field
-  is to embed a library written in foreign language and/or building with another
-  build system. It is not for casual uses, see the `re2 library
-  <https://github.com/janestreet/re2>`__ for an example of use
-
 - ``(modules_without_implementation <modules>)`` specifies a list of
   modules that have only a ``.mli`` or ``.rei`` but no ``.ml`` or
   ``.re`` file. Such modules are usually referred as *mli only
@@ -617,15 +610,13 @@ Executables can also be linked as object or shared object files. See
   ``executable`` stanza will cause Dune to copy the ``.exe`` files to
   the source tree and ``dune clean`` to delete them
 
-- ``(c_names (<names>))``, if your executable needs C stubs, you must list the C
-  files in this field, without the ``.c`` extension
-
-- ``(cxx_names (<names>))`` is the same as ``c_names`` but for C++ stubs
-
-- ``(c_flags <flags>)`` specifies the compilation flags for C stubs, using the
-  :ref:`ordered-set-language`. This field supports ``(:include ...)`` forms
+- ``(foreign_stubs <foreign-stubs-spec>)`` specifies foreign source
+  files, e.g. C or C++ stubs, to be linked into the executable. See the
+  section :ref:`foreign-sources-and-archives` for more details.
 
-- ``(cxx_flags <flags>)`` is the same as ``c_flags`` but for C++ stubs
+- ``(foreign_archives <foreign-archives-list>)`` specifies archives of
+  foreign object files to be linked into the executable. See the section
+  :ref:`foreign-archives` for more details.
 
 - ``(forbidden_libraries <libraries>)`` ensures that the given
   libraries are not linked in the resulting executable. If they end up

doc/foreign-code.rst

@@ -24,24 +24,24 @@ Adding C/C++ stubs to an OCaml library
 ======================================
 
 To add C stubs to an OCaml library, simply list the C files without
-the ``.c`` extension via the ``c_names`` field of the :ref:`library`
-stanza. For instance:
+the ``.c`` extension in the :ref:`foreign-stubs` field. For instance:
 
 .. code:: scheme
 
           (library
            (name mylib)
-           (c_names file1 file2))
+           (foreign_stubs (language c) (names file1 file2)))
 
-Similarly, you can add C++ stubs to an OCaml library by listing them
-without the ``.cpp`` extension via the ``cxx_names`` field.
+You can also add C++ stubs to an OCaml library by specifying
+``(language cxx)`` instead.
 
 Dune is currently not flexible regarding the extension of the C/C++
-source files. They have to be ``.c`` and ``.cpp``. If you have source
-files that that do not follow this extension and you want to build
-them with Dune, you need to rename them first. Alternatively, you can
-use the :ref:`foreign build sandboxing <foreign-sandboxing>` method
-described bellow.
+source files. They have to be ``.c`` for C files and ``.cpp``, ``.cc``
+or ``.cxx`` for C++ files. If you have source files with other
+extensions and you want to build them with Dune, you need to rename
+them first. Alternatively, you can use the
+:ref:`foreign build sandboxing <foreign-sandboxing>` method described
+below.
 
 Header files
 ------------
@@ -80,12 +80,9 @@ To do that, follow the following procedure:
   :ref:`data_only_dirs <dune-data_only_dirs>` stanza
 - write a custom rule that:
 
-  - depend on this directory recursively via :ref:`source_tree <source_tree>`
-  - invoke the external build system
-  - copy the C archive files (``.a``, ``.so``, ...) in main library
-    directory with a specific names (see bellow)
-- *attach* the C archive files to an OCaml library via the
-  :ref:`self_build_stubs_archive <self_build_stubs_archive>` field
+  - depends on this directory recursively via :ref:`source_tree <source_tree>`
+  - invokes the external build system
+- *attach* the C archive files to an OCaml library via :ref:`foreign-archives`.
 
 For instance, let's assume that you want to build a C library
 ``libfoo`` using ``libfoo``'s own build system and attach it to an
@@ -106,16 +103,8 @@ writing the following code ``src/dune``:
 
           (rule
            (deps (source_tree libfoo))
-           (targets libfoo_stubs.a dllfoo_stubs.so)
-           (action (progn
-                    (chdir libfoo (run make)))
-                    (copy libfoo/libfoo.a libfoo_stubs.a)
-                    (copy libfoo/libfoo.so dllfoo_stubs.so)))
-
-Note that the rule copies the files to ``libfoo_stubs.a`` and
-``dllfoo_stubs.so``. It is important that the files produced are
-named ``lib<ocaml-lib-name>_stubs.a`` and
-``dll<ocaml-lib-name>_stubs.so``.
+           (targets libfoo.a dllfoo.so)
+           (action (chdir libfoo (run make))))
 
 The last step is to attach these archives to an OCaml library as
 follows:
@@ -124,7 +113,7 @@ follows:
 
           (library
            (name bar)
-           (self_build_stubs_archive foo))
+           (foreign_archives libfoo/foo))
 
 Then, whenever you use the ``bar`` library, you will also be able to
 use C functions from ``libfoo``.

editor-integration/emacs/dune.el

@@ -63,7 +63,8 @@
                "ocamllex" "ocamlyacc" "menhir" "alias" "install"
                "copy_files" "copy_files#" "include" "tests" "test" "dirs"
                "env" "ignored_subdirs" "include_subdirs" "data_only_dirs"
-               "documentation" "cinaps" "coqlib" "coq.theory" "coq.pp")
+               "documentation" "cinaps" "coqlib" "coq.theory" "coq.pp"
+               "foreign_library")
              ) "\\(?:\\_>\\|[[:space:]]\\)"))
   "Stanzas in dune files.")
 
@@ -72,9 +73,9 @@
     (regexp-opt
      '("name" "public_name" "synopsis" "modules" "libraries" "wrapped"
        "preprocess" "preprocessor_deps" "optional" "c_names" "cxx_names"
-       "install_c_headers" "modes" "no_dynlink" "kind"
-       "ppx_runtime_libraries" "virtual_deps" "js_of_ocaml" "flags"
-       "ocamlc_flags" "ocamlopt_flags" "library_flags" "c_flags"
+       "foreign_stubs" "foreign_archives" "install_c_headers" "modes"
+       "no_dynlink" "kind" "ppx_runtime_libraries" "virtual_deps" "js_of_ocaml"
+       "flags" "ocamlc_flags" "ocamlopt_flags" "library_flags" "c_flags"
        "cxx_flags" "c_library_flags" "self_build_stubs_archive" "inline_tests"
        "modules_without_implementation" "private_modules"
        ;; + special_builtin_support
@@ -87,6 +88,8 @@
        ;; + for "executable" and "executables":
        "package" "link_flags" "link_deps" "names" "public_names" "variants"
        "forbidden_libraries"
+       ;; + for "foreign_library" and "foreign_stubs":
+       "archive_name" "language" "names" "flags" "include_dirs" "extra_deps"
        ;; + for "rule":
        "targets" "action" "deps" "mode" "fallback" "locks"
        ;; + for "menhir":