Adds dune-action-plugin package.

This commit adds `dune-action-plugin` package and library.

* New packages: `dune-private-libs` and `dune-action-plugin`.
* New construction in action definition - `dynamic-run`.
* New variant in `Action.t` type - `Dynamic_run`.
* Changes in `Action_exec` and `build_system.ml`.

Tests:
* `test/blackbox-tests/test-cases-with-libs/dune-action-plugin`
* `test/expect-tests/dune_action_plugin`

Signed-off-by: Jakub Staron <jstaron@janestreet.com>

bootstrap.ml

@@ -36,6 +36,7 @@ type subdirs =
 let dirs =
   [ ("src/stdune/result", Some "Dune_result", No)
   ; ("src/stdune/caml", Some "Dune_caml", No)
+  ; ("src/dune_action_plugin", Some "Dune_action_plugin", No)
   ; ("src/stdune", Some "Stdune", No)
   ; ("src/fiber", Some "Fiber", No)
   ; ("src/xdg", Some "Xdg", No)

doc/concepts.rst

@@ -595,13 +595,18 @@ automatically handled by dune.
 The DSL is currently quite limited, so if you want to do something complicated
 it is recommended to write a small OCaml program and use the DSL to invoke it.
 You can use `shexp <https://github.com/janestreet/shexp>`__ to write portable
-scripts or :ref:`configurator` for configuration related tasks.
+scripts or :ref:`configurator` for configuration related tasks. You can also
+use :ref:`dune-action-plugin` to express program dependencies directly in the
+source code.
 
 The following constructions are available:
 
 - ``(run <prog> <args>)`` to execute a program. ``<prog>`` is resolved
   locally if it is available in the current workspace, otherwise it is
   resolved using the ``PATH``
+- ``(dynamic-run <prog> <args>)`` to execute a program that was linkied
+  against ``dune-action-plugin`` library. ``<prog>`` is resolved in
+  the same way as in ``run``
 - ``(chdir <dir> <DSL>)`` to change the current directory
 - ``(setenv <var> <value> <DSL>)`` to set an environment variable
 - ``(with-<outputs>-to <file> <DSL>)`` to redirect the output to a file, where
@@ -696,6 +701,15 @@ of your project. What you should write instead is:
      (deps   blah.mll)
      (action (chdir %{workspace_root} (run ocamllex -o %{target} %{deps}))))
 
+.. _dune-action-plugin:
+
+Dune action plugin
+==================
+
+``Dune-action-plugin`` provides a monadic interface to express program
+dependencies directly inside the source code. Programs using this feature
+should be declared using ``dynamic-run`` construction instead of usual ``run``.
+
 Sandboxing
 ==========
 

dune-action-plugin.opam

@@ -0,0 +1,29 @@
+# This file is generated by dune, edit dune-project instead
+opam-version: "2.0"
+synopsis: "Applicative and monadic interface for declaring dependencies."
+description: ""
+maintainer: ["Jane Street Group, LLC <opensource@janestreet.com>"]
+authors: ["Jane Street Group, LLC <opensource@janestreet.com>"]
+license: "MIT"
+homepage: "https://github.com/ocaml/dune"
+doc: "https://dune.readthedocs.io/"
+bug-reports: "https://github.com/ocaml/dune/issues"
+depends: [
+  "dune" {>= "2.0"}
+  "ocaml" {>= "4.02"}
+]
+build: [
+  ["dune" "subst"] {pinned}
+  [
+    "dune"
+    "build"
+    "-p"
+    name
+    "-j"
+    jobs
+    "@install"
+    "@runtest" {with-test}
+    "@doc" {with-doc}
+  ]
+]
+dev-repo: "git+https://github.com/ocaml/dune.git"

dune-private-libs.opam

@@ -0,0 +1,29 @@
+# This file is generated by dune, edit dune-project instead
+opam-version: "2.0"
+synopsis: "Dune private libraries. Do not use!"
+description: "Dune private libraries. Do not use!"
+maintainer: ["Jane Street Group, LLC <opensource@janestreet.com>"]
+authors: ["Jane Street Group, LLC <opensource@janestreet.com>"]
+license: "MIT"
+homepage: "https://github.com/ocaml/dune"
+doc: "https://dune.readthedocs.io/"
+bug-reports: "https://github.com/ocaml/dune/issues"
+depends: [
+  "dune" {>= "2.0"}
+  "ocaml" {>= "4.02"}
+]
+build: [
+  ["dune" "subst"] {pinned}
+  [
+    "dune"
+    "build"
+    "-p"
+    name
+    "-j"
+    jobs
+    "@install"
+    "@runtest" {with-test}
+    "@doc" {with-doc}
+  ]
+]
+dev-repo: "git+https://github.com/ocaml/dune.git"

dune-project

@@ -53,3 +53,17 @@ versions.  It supports reporting the version from the version control
 system during development to get an precise reference of when the
 executable was built.
 "))
+
+(package
+ (name dune-private-libs)
+ (depends
+  (ocaml (>= 4.02)))
+ (synopsis "Dune private libraries. Do not use!")
+ (description "Dune private libraries. Do not use!"))
+
+(package
+ (name dune-action-plugin)
+ (depends
+  (ocaml (>= 4.02)))
+ (synopsis "Applicative and monadic interface for declaring dependencies.")
+ (description ""))

src/configurator/dune

@@ -4,7 +4,7 @@
  (name configurator)
  (public_name dune.configurator)
  (preprocess future_syntax)
- (libraries stdune ocaml_config dune_lang dune_caml dune._result)
+ (libraries stdune ocaml_config dune_lang dune_caml dune-private-libs.result)
  (flags
   (:standard
    -safe-string

src/dune/action.ml

@@ -151,6 +151,7 @@ let fold_one_step t ~init:acc ~f =
     f acc t
   | Progn l -> List.fold_left l ~init:acc ~f
   | Run _
+   |Dynamic_run _
    |Echo _
    |Cat _
    |Copy _
@@ -180,6 +181,32 @@ let chdirs =
   in
   fun t -> loop Path.Set.empty t
 
+let rec is_dynamic = function
+  | Dynamic_run _ -> true
+  | Chdir (_, t)
+   |Setenv (_, _, t)
+   |Redirect_out (_, _, t)
+   |Redirect_in (_, _, t)
+   |Ignore (_, t) ->
+    is_dynamic t
+  | Progn l -> List.exists l ~f:is_dynamic
+  | Run _
+   |System _
+   |Bash _
+   |Echo _
+   |Cat _
+   |Copy _
+   |Symlink _
+   |Copy_and_add_line_directive _
+   |Write_file _
+   |Rename _
+   |Remove_tree _
+   |Diff _
+   |Mkdir _
+   |Digest_files _
+   |Merge_files_into _ ->
+    false
+
 let prepare_managed_paths ~link ~sandboxed deps ~eval_pred =
   let steps =
     Path.Set.fold (Dep.Set.paths deps ~eval_pred) ~init:[] ~f:(fun path acc ->
@@ -253,6 +280,7 @@ let is_useful_to_sandbox =
     | Digest_files _ -> false
     | Merge_files_into _ -> false
     | Run _ -> true
+    | Dynamic_run _ -> true
     | System _ -> true
     | Bash _ -> true
   in

src/dune/action.mli

@@ -68,6 +68,9 @@ val for_shell : t -> For_shell.t
 (** Return the list of directories the action chdirs to *)
 val chdirs : t -> Path.Set.t
 
+(** Checks, if action contains a [Dynamic_run]. *)
+val is_dynamic : t -> bool
+
 (** Ast where programs are not yet looked up in the PATH *)
 module Unresolved : sig
   type action = t

src/dune/action_ast.ml

@@ -57,6 +57,10 @@ struct
             , let+ prog = Program.decode
               and+ args = repeat String.decode in
               Run (prog, args) )
+          ; ( "dynamic-run"
+            , let+ prog = Program.decode
+              and+ args = repeat String.decode in
+              Dynamic_run (prog, args) )
           ; ( "chdir"
             , let+ dn = path
               and+ t = t in
@@ -129,6 +133,8 @@ struct
     let target = Target.encode in
     function
     | Run (a, xs) -> List (atom "run" :: program a :: List.map xs ~f:string)
+    | Dynamic_run (a, xs) ->
+      List (atom "run_dynamic" :: program a :: List.map xs ~f:string)
     | Chdir (a, r) -> List [ atom "chdir"; path a; encode r ]
     | Setenv (k, v, r) -> List [ atom "setenv"; string k; string v; encode r ]
     | Redirect_out (outputs, fn, r) ->

src/dune/action_dune_lang.ml

@@ -29,6 +29,47 @@ include Action_ast.Make (String_with_vars) (String_with_vars)
           (Uast)
 module Mapper = Action_mapper.Make (Uast) (Uast)
 
+let ensure_at_most_one_dynamic_run ~loc action =
+  let rec loop : t -> bool = function
+    | Dynamic_run _ -> true
+    | Chdir (_, t)
+     |Setenv (_, _, t)
+     |Redirect_out (_, _, t)
+     |Redirect_in (_, _, t)
+     |Ignore (_, t) ->
+      loop t
+    | Run _
+     |Echo _
+     |Cat _
+     |Copy _
+     |Symlink _
+     |Copy_and_add_line_directive _
+     |System _
+     |Bash _
+     |Write_file _
+     |Rename _
+     |Remove_tree _
+     |Mkdir _
+     |Digest_files _
+     |Diff _
+     |Merge_files_into _ ->
+      false
+    | Progn ts ->
+      List.fold_left ts ~init:false ~f:(fun acc t ->
+          let have_dyn = loop t in
+          if acc && have_dyn then
+            User_error.raise ~loc
+              [ Pp.text
+                  "Multiple 'dynamic-run' commands within single action are \
+                   not supported."
+              ]
+          else
+            acc || have_dyn)
+  in
+  ignore (loop action)
+
+let validate ~loc t = ensure_at_most_one_dynamic_run ~loc t
+
 let remove_locs =
   let dir = String_with_vars.make_text Loc.none "" in
   let f_program ~dir:_ = String_with_vars.remove_locs in
@@ -42,7 +83,12 @@ let compare_no_locs t1 t2 = compare (remove_locs t1) (remove_locs t2)
 open Dune_lang.Decoder
 
 let decode =
-  if_list ~then_:decode
+  if_list
+    ~then_:
+      ( located decode
+      >>| fun (loc, action) ->
+      validate ~loc action;
+      action )
     ~else_:
       ( loc
       >>| fun loc ->

src/dune/action_dune_lang.mli

@@ -11,6 +11,9 @@ include
 
 include Dune_lang.Conv.S with type t := t
 
+(** Raises User_error on invalid action. *)
+val validate : loc:Loc.t -> t -> unit
+
 include
   Action_intf.Helpers
     with type t := t