diff --git a/bin/alias.ml b/bin/alias.ml index 6dc2f89e03a..e03e4b0fc45 100644 --- a/bin/alias.ml +++ b/bin/alias.ml @@ -76,7 +76,8 @@ let dep_on_alias_multi_contexts ~dir ~name ~contexts = let context_to_alias_expansion ctx = let ctx_dir = Context_name.build_dir ctx in let dir = Path.Build.(append_source ctx_dir dir) in - Action_builder.alias (Alias.make ~dir name) + Build_system.record_deps + (Dep.Set.singleton (Dep.alias (Dune_engine.Alias.make ~dir name))) in Action_builder.all_unit (List.map contexts ~f:context_to_alias_expansion) ;; diff --git a/bin/build_cmd.ml b/bin/build_cmd.ml index 18b1982a775..56a181d4dce 100644 --- a/bin/build_cmd.ml +++ b/bin/build_cmd.ml @@ -52,7 +52,9 @@ let run_build_system ~common ~request = let toplevel_cell, toplevel = Memo.Lazy.Expert.create ~name:"toplevel" (fun () -> let open Memo.O in - let+ (), (_ : Dep.Fact.t Dep.Map.t) = Action_builder.run request Eager in + let+ (), (_ : Dep.Facts.t) = + Action_builder.evaluate_and_collect_facts request + in ()) in let* res = run ~toplevel in diff --git a/bin/target.ml b/bin/target.ml index b62b52e3537..85ada309287 100644 --- a/bin/target.ml +++ b/bin/target.ml @@ -13,7 +13,7 @@ let request targets = acc >>> match (target : Request.t) with - | File path -> Action_builder.path path + | File path -> Build_system.record_deps (Dep.Set.singleton (Dep.file path)) | Alias a -> Alias.request a) ;; diff --git a/src/dune_engine/action_builder.ml b/src/dune_engine/action_builder.ml index 0ee619f4394..f50dafb7a73 100644 --- a/src/dune_engine/action_builder.ml +++ b/src/dune_engine/action_builder.ml @@ -1,341 +1,276 @@ open Import -include Action_builder0 -open O -open struct - module List = Stdune.List -end - -let register_action_deps : type a. a eval_mode -> Dep.Set.t -> a Dep.Map.t Memo.t = - fun mode deps -> - match mode with - | Eager -> Build_system.build_deps deps - | Lazy -> Memo.return deps -;; +type 'a eval_mode = + | Lazy : Dep.Set.t eval_mode + | Eager : Dep.Facts.t eval_mode -let dyn_memo_deps deps = - of_thunk - { f = - (fun mode -> - let open Memo.O in - let* deps, paths = deps in - let+ deps = register_action_deps mode deps in - paths, deps) - } -;; +type 'a thunk = { f : 'm. 'm eval_mode -> ('a * 'm) Memo.t } [@@unboxed] -let deps d = dyn_memo_deps (Memo.return (d, ())) -let dep d = deps (Dep.Set.singleton d) +module Deps_or_facts = struct + let empty : type m. m eval_mode -> m = function + | Lazy -> Dep.Set.empty + | Eager -> Dep.Facts.empty + ;; -let dyn_deps t = - of_thunk - { f = - (fun mode -> - let open Memo.O in - let* (x, deps), deps_x = run t mode in - let+ deps = register_action_deps mode deps in - x, Deps_or_facts.union mode deps deps_x) - } -;; + let return : type a m. a -> m eval_mode -> a * m = fun a mode -> a, empty mode -let path p = deps (Dep.Set.singleton (Dep.file p)) -let paths ps = deps (Dep.Set.of_files ps) -let path_set ps = deps (Dep.Set.of_files_set ps) + let union : type m. m eval_mode -> m -> m -> m = + fun mode a b -> + match mode with + | Lazy -> Dep.Set.union a b + | Eager -> Dep.Facts.union a b + ;; -let paths_matching - : type a. File_selector.t -> a eval_mode -> (Path.Set.t * a Dep.Map.t) Memo.t - = - fun g mode -> - let open Memo.O in - match mode with - | Eager -> - let+ files = Build_system.build_pred g in - ( Path.Map.keys (Dep.Fact.Files.paths files) |> Path.Set.of_list - , Dep.Map.singleton (Dep.file_selector g) (Dep.Fact.file_selector g files) ) - | Lazy -> - let+ files = Build_system.eval_pred g in - files, Dep.Set.singleton (Dep.file_selector g) -;; + let union_all : type m. m eval_mode -> m list -> m = + fun mode list -> + match mode with + | Lazy -> Dep.Set.union_all list + | Eager -> Dep.Facts.union_all list + ;; +end -let paths_matching ~loc:_ g = of_thunk { f = (fun mode -> paths_matching g mode) } -let paths_matching_unit ~loc g = ignore (paths_matching ~loc g) -let dyn_paths paths = dyn_deps (paths >>| fun (x, paths) -> x, Dep.Set.of_files paths) -let dyn_paths_unit paths = dyn_deps (paths >>| fun paths -> (), Dep.Set.of_files paths) +module T = struct + open Memo.O -let dyn_path_set paths = - dyn_deps (paths >>| fun (x, paths) -> x, Dep.Set.of_files_set paths) -;; + module M = struct + type 'a t = 'a thunk -let dyn_path_set_reuse paths = - dyn_deps (paths >>| fun paths -> paths, Dep.Set.of_files_set paths) -;; + let return x = { f = (fun mode -> Memo.return (Deps_or_facts.return x mode)) } -let env_var s = deps (Dep.Set.singleton (Dep.env s)) -let alias a = dep (Dep.alias a) - -let contents = - let read_file = - Memo.exec - (Memo.create_with_store - "Action_builder.contents" - ~store:(module Path.Table) - ~input:(module Path) - ~cutoff:String.equal - (fun p -> Build_system.read_file p ~f:Io.read_file)) - in - fun p -> - of_thunk + let map t ~f = { f = - (fun _mode -> - let open Memo.O in - let+ x = read_file p in - x, Dep.Map.empty) + (fun mode -> + let+ x, deps = t.f mode in + f x, deps) } -;; + ;; -let lines_of p = contents p >>| String.split_lines + let bind t ~f = + { f = + (fun mode -> + let* x, deps1 = t.f mode in + let+ y, deps2 = (f x).f mode in + y, Deps_or_facts.union mode deps1 deps2) + } + ;; + end -let read_sexp p = - let+ s = contents p in - Dune_sexp.Parser.parse_string s ~fname:(Path.to_string p) ~mode:Single -;; + include M -let if_file_exists p ~then_ ~else_ = - of_thunk + let both x y = { f = (fun mode -> - let open Memo.O in - Build_system.file_exists p - >>= function - | true -> run then_ mode - | false -> run else_ mode) - } -;; - -let file_exists p = if_file_exists p ~then_:(return true) ~else_:(return false) - -let paths_existing paths = - all_unit - (List.map paths ~f:(fun file -> - if_file_exists file ~then_:(path file) ~else_:(return ()))) -;; - -let fail x = - let+ () = return () in - x.fail () -;; - -(* CR-someday amokhov: The set of targets is accumulated using information from - multiple sources by calling [Targets.combine], which performs set union and - hence duplicate declarations of the very same target can go unnoticed. I - think such redeclarations are not erroneous but are merely redundant; perhaps - we should detect and disallow them. *) -module With_targets = struct - type nonrec 'a t = - { build : 'a t - ; targets : Targets.t + let+ x, deps1 = x.f mode + and+ y, deps2 = y.f mode in + (x, y), Deps_or_facts.union mode deps1 deps2) } + ;; - let map_build t ~f = { t with build = f t.build } - let return x = { build = return x; targets = Targets.empty } - - let add t ~file_targets = - { build = t.build - ; targets = - Targets.combine - t.targets - (Targets.Files.create (Path.Build.Set.of_list file_targets)) + let all xs = + { f = + (fun mode -> + let+ res = Memo.parallel_map xs ~f:(fun x -> x.f mode) in + let res, facts = List.split res in + res, Deps_or_facts.union_all mode facts) } ;; - let add_directories t ~directory_targets = - { build = t.build - ; targets = - Targets.combine - t.targets - (Targets.create - ~dirs:(Path.Build.Set.of_list directory_targets) - ~files:Path.Build.Set.empty) + let of_memo memo = + { f = + (fun mode -> + let+ x = memo in + x, Deps_or_facts.empty mode) } ;; - let map { build; targets } ~f = { build = map build ~f; targets } - - let map2 x y ~f = - { build = map2 x.build y.build ~f; targets = Targets.combine x.targets y.targets } + let record res (deps : Dep.Set.t) ~f = + let f : type m. m eval_mode -> (_ * m) Memo.t = + fun mode -> + let open Memo.O in + match mode with + | Lazy -> Memo.return (res, deps) + | Eager -> + let+ facts = Dep.Facts.record_facts deps ~f in + res, facts + in + { f } ;; - let both x y = - { build = both x.build y.build; targets = Targets.combine x.targets y.targets } + let record_success memo = + let f : type m. m eval_mode -> (unit * m) Memo.t = + fun mode -> + let open Memo.O in + match mode with + | Lazy -> Memo.return ((), Dep.Set.empty) + | Eager -> + let+ () = memo in + (), Dep.Facts.empty + in + { f } ;; - let seq x y = - { build = x.build >>> y.build; targets = Targets.combine x.targets y.targets } - ;; + module Expert = struct + let record_dep_on_source_file_exn res ?loc (src_path : Path.Source.t) = + let f : type m. m eval_mode -> (_ * m) Memo.t = + fun mode -> + let (path : Path.t) = Path.source src_path in + let dep = Dep.file path in + match mode with + | Lazy -> Memo.return (res, Dep.Set.singleton dep) + | Eager -> + let open Memo.O in + let+ digest = + Fs_memo.file_digest_exn ?loc (Path.Outside_build_dir.In_source_dir src_path) + in + res, Dep.Facts.singleton dep (Dep.Fact.file path digest) + in + { f } + ;; + end module O = struct - let ( >>> ) = seq + let ( >>> ) a b = + { f = + (fun mode -> + let+ ((), deps_a), (b, deps_b) = + Memo.fork_and_join (fun () -> a.f mode) (fun () -> b.f mode) + in + b, Deps_or_facts.union mode deps_a deps_b) + } + ;; + + let ( >>= ) t f = bind t ~f let ( >>| ) t f = map t ~f let ( and+ ) = both - let ( let+ ) a f = map ~f a + let ( and* ) = both + let ( let+ ) t f = map t ~f + let ( let* ) t f = bind t ~f end - open O + module List = struct + include Monad.List (struct + include M + include Monad.Make (M) + end) - let all xs = - match xs with - | [] -> return [] - | xs -> - let build, targets = - List.fold_left xs ~init:([], Targets.empty) ~f:(fun (builds, targets) x -> - x.build :: builds, Targets.combine x.targets targets) - in - { build = all (List.rev build); targets } - ;; - - let write_file_dyn ?(perm = Action.File_perm.Normal) fn s = - add - ~file_targets:[ fn ] - (let+ s = s in - Action.Full.make (Action.Write_file (fn, perm, s))) - ;; + let map l ~f = + { f = + (fun mode -> + let+ res = Memo.parallel_map l ~f:(fun x -> (f x).f mode) in + let res, deps = List.split res in + res, Deps_or_facts.union_all mode deps) + } + ;; - let memoize name t = { build = memoize name t.build; targets = t.targets } + let concat_map l ~f = + { f = + (fun mode -> + let+ res = Memo.parallel_map l ~f:(fun x -> (f x).f mode) in + let res, deps = List.split res in + List.concat res, Deps_or_facts.union_all mode deps) + } + ;; + end end -let with_targets build ~targets : _ With_targets.t = { build; targets } - -let with_file_targets build ~file_targets : _ With_targets.t = - { build; targets = Targets.Files.create (Path.Build.Set.of_list file_targets) } -;; - -let with_no_targets build : _ With_targets.t = { build; targets = Targets.empty } +include T +open O -let write_file ?(perm = Action.File_perm.Normal) fn s = - with_file_targets - ~file_targets:[ fn ] - (return (Action.Full.make (Action.Write_file (fn, perm, s)))) -;; +open struct + module List = Stdune.List +end -let write_file_dyn ?(perm = Action.File_perm.Normal) fn s = - with_file_targets - ~file_targets:[ fn ] - (let+ s = s in - Action.Full.make (Action.Write_file (fn, perm, s))) -;; +let evaluate_and_collect_deps t = t.f Lazy +let evaluate_and_collect_facts t = t.f Eager -let with_stdout_to ?(perm = Action.File_perm.Normal) fn t = - with_targets - ~targets:(Targets.File.create fn) - (let+ (act : Action.Full.t) = t in - Action.Full.map act ~f:(Action.with_stdout_to ~perm fn)) +let force_lazy_or_eager + : type a m. + m eval_mode + -> (a * Dep.Set.t) Memo.Lazy.t Lazy.t + -> (a * Dep.Facts.t) Memo.Lazy.t + -> (a * m) Memo.t + = + fun mode lazy_ eager -> + match mode with + | Lazy -> Memo.Lazy.force (Lazy.force lazy_) + | Eager -> Memo.Lazy.force eager ;; -let copy ~src ~dst = - with_file_targets - ~file_targets:[ dst ] - (path src >>> return (Action.Full.make (Action.Copy (src, dst)))) +let memoize ?cutoff name t = + let lazy_ : ('a * Dep.Set.t) Memo.Lazy.t Lazy.t = + lazy + (let cutoff = + Option.map cutoff ~f:(fun equal x y -> Tuple.T2.equal equal Dep.Set.equal x y) + in + Memo.lazy_ ?cutoff ~name:(name ^ "(lazy)") (fun () -> t.f Lazy)) + in + (* Unlike [lazy_], [eager] doesn't have the outer [Lazy.t] wrapper because most [Eager] + nodes end up getting forced during every build. *) + let eager : ('a * Dep.Facts.t) Memo.Lazy.t = + let cutoff = + Option.map cutoff ~f:(fun equal x y -> Tuple.T2.equal equal Dep.Facts.equal x y) + in + Memo.lazy_ ?cutoff ~name (fun () -> t.f Eager) + in + { f = (fun mode -> force_lazy_or_eager mode lazy_ eager) } ;; -let symlink ~src ~dst = - with_file_targets - ~file_targets:[ dst ] - (path src >>> return (Action.Full.make (Action.Symlink (src, dst)))) +let map2 x y ~f = + let+ x = x + and+ y = y in + f x y ;; -let symlink_dir ~src ~dst = - with_targets - ~targets: - (Targets.create ~files:Path.Build.Set.empty ~dirs:(Path.Build.Set.singleton dst)) - (path src >>> return (Action.Full.make (Action.Symlink (src, dst)))) +let all_unit (xs : unit t list) = + { f = + (fun mode -> + let open Memo.O in + let+ res = Memo.parallel_map xs ~f:(fun x -> x.f mode) in + let deps = List.map res ~f:snd in + (), Deps_or_facts.union_all mode deps) + } ;; -let create_file ?(perm = Action.File_perm.Normal) fn = - with_file_targets - ~file_targets:[ fn ] - (return (Action.Full.make (Action.Redirect_out (Stdout, fn, perm, Action.empty)))) +type ('input, 'output) memo = + { lazy_ : ('input, 'output * Dep.Set.t) Memo.Table.t Lazy.t + ; eager : ('input, 'output * Dep.Facts.t) Memo.Table.t Lazy.t + } + +let create_memo name ~input ?cutoff ?human_readable_description f = + let lazy_ = + lazy + (let cutoff = + Option.map cutoff ~f:(fun f (a, deps1) (b, deps2) -> + f a b && Dep.Set.equal deps1 deps2) + in + let name = name ^ "(lazy)" in + Memo.create name ~input ?cutoff ?human_readable_description (fun x -> (f x).f Lazy)) + and eager = + lazy + (let cutoff = + Option.map cutoff ~f:(fun f (a, facts1) (b, facts2) -> + f a b && Dep.Facts.equal facts1 facts2) + in + Memo.create name ~input ?cutoff ?human_readable_description (fun x -> + (f x).f Eager)) + in + { lazy_; eager } ;; -let progn ts = - let open With_targets.O in - With_targets.all ts >>| Action.Full.reduce +let exec_memo : type i o m. (i, o) memo -> i -> m eval_mode -> (o * m) Memo.t = + fun memo i mode -> + match mode with + | Lazy -> Memo.exec (Lazy.force memo.lazy_) i + | Eager -> Memo.exec (Lazy.force memo.eager) i ;; -let dyn_of_memo_deps t = dyn_deps (dyn_of_memo t) - -module Alias_status = struct - module T = struct - type t = - | Defined - | Not_defined - - let empty : t = Not_defined - - let combine : t -> t -> t = - fun x y -> - match x, y with - | _, Defined | Defined, _ -> Defined - | Not_defined, Not_defined -> Not_defined - ;; - end - - include T - include Monoid.Make (T) -end - -module Alias_build_info = struct - type t = - { alias_status : Alias_status.t - ; allowed_build_only_subdirs : Filename.Set.t - } - - let of_dir_set ~status dirs = - let allowed_build_only_subdirs = - match Dir_set.toplevel_subdirs dirs with - | Infinite -> Filename.Set.empty - | Finite sub_dirs -> sub_dirs - in - { alias_status = status; allowed_build_only_subdirs } - ;; -end +let exec_memo m i = { f = (fun mode -> exec_memo m i mode) } -let dep_on_alias_build_info_if_exists alias = - of_thunk - { f = - (fun mode -> - let open Memo.O in - Load_rules.load_dir ~dir:(Path.build (Alias.dir alias)) - >>= function - | Source _ | External _ -> - Code_error.raise "Alias in a non-build dir" [ "alias", Alias.to_dyn alias ] - | Build { aliases; allowed_subdirs; rules_here = _ } -> - (match Alias.Name.Map.find aliases (Alias.name alias) with - | None -> - Memo.return - ( Alias_build_info.of_dir_set ~status:Not_defined allowed_subdirs - , Dep.Map.empty ) - | Some _ -> - let deps = Dep.Set.singleton (Dep.alias alias) in - let+ deps = register_action_deps mode deps in - Alias_build_info.of_dir_set ~status:Defined allowed_subdirs, deps) - | Build_under_directory_target _ -> - Memo.return - ( Alias_build_info.of_dir_set ~status:Not_defined Dir_set.empty - , Dep.Map.empty )) - } +let goal t = + { f = + (fun mode -> + let open Memo.O in + let+ a, _facts_are_irrelevant_for_goals = t.f mode in + a, Deps_or_facts.empty mode) + } ;; - -module Alias_rec (Traverse : sig - val traverse - : Path.Build.t - -> f:(path:Path.Build.t -> Alias_build_info.t t) - -> Alias_status.t t - end) = -struct - open Traverse - - let dep_on_alias_rec name dir = - let f ~path = dep_on_alias_build_info_if_exists (Alias.make ~dir:path name) in - traverse dir ~f - ;; -end diff --git a/src/dune_engine/action_builder.mli b/src/dune_engine/action_builder.mli index b5fc612e008..e42ec70beb0 100644 --- a/src/dune_engine/action_builder.mli +++ b/src/dune_engine/action_builder.mli @@ -1,193 +1,92 @@ -(** Action builder *) - -open! Import -include module type of Action_builder0 - -module With_targets : sig - type 'a build := 'a t - - type nonrec 'a t = - { build : 'a t - ; targets : Targets.t - } - - val map_build : 'a t -> f:('a build -> 'b build) -> 'b t - val return : 'a -> 'a t - val add : 'a t -> file_targets:Path.Build.t list -> 'a t - val add_directories : 'a t -> directory_targets:Path.Build.t list -> 'a t - val map : 'a t -> f:('a -> 'b) -> 'b t - val map2 : 'a t -> 'b t -> f:('a -> 'b -> 'c) -> 'c t - - val write_file_dyn - : ?perm:Action.File_perm.t - -> Path.Build.t - -> string t - -> Action.Full.t t - - val all : 'a t list -> 'a list t - - (** [memoize name t] is an action builder that behaves like [t] except that - its result is computed only once. *) - val memoize : string -> 'a t -> 'a t - - module O : sig - val ( >>> ) : unit t -> 'a t -> 'a t - val ( >>| ) : 'a t -> ('a -> 'b) -> 'b t - val ( let+ ) : 'a t -> ('a -> 'b) -> 'b t - val ( and+ ) : 'a t -> 'b t -> ('a * 'b) t - end +open Import + +type 'a t + +module O : sig + val ( >>> ) : unit t -> 'a t -> 'a t + val ( >>= ) : 'a t -> ('a -> 'b t) -> 'b t + val ( >>| ) : 'a t -> ('a -> 'b) -> 'b t + val ( let* ) : 'a t -> ('a -> 'b t) -> 'b t + val ( and* ) : 'a t -> 'b t -> ('a * 'b) t + val ( let+ ) : 'a t -> ('a -> 'b) -> 'b t + val ( and+ ) : 'a t -> 'b t -> ('a * 'b) t end -(** Add targets to an action builder, turning a target-less [Action_builder.t] - into [Action_builder.With_targets.t]. *) -val with_targets : 'a t -> targets:Targets.t -> 'a With_targets.t - -(** Like [with_targets] but specifies a list of file targets. *) -val with_file_targets : 'a t -> file_targets:Path.Build.t list -> 'a With_targets.t - -(** Create a value of [With_targets.t] with the empty set of targets. *) -val with_no_targets : 'a t -> 'a With_targets.t - -(** CR-someday diml: this API is not great, what about: - - {[ - module Action_with_deps : sig - type t - val add_file_dependency : t -> Path.t -> t - end - - (** Same as - [t >>> arr (fun x -> Action_with_deps.add_file_dependency x p)] - but better as [p] is statically known *) - - val record_dependency - : Path.t - -> ('a, Action_with_deps.t) t - -> ('a, Action_with_deps.t) t - ]} *) - -(** [path p] records [p] as a file that is read by the action produced by the - action builder. *) -val path : Path.t -> unit t - -val dep : Dep.t -> unit t -val deps : Dep.Set.t -> unit t -val dyn_deps : ('a * Dep.Set.t) t -> 'a t -val paths : Path.t list -> unit t -val path_set : Path.Set.t -> unit t - -(** Evaluate a predicate against all targets and record all the matched files as - dependencies of the action produced by the action builder. *) -val paths_matching : loc:Loc.t -> File_selector.t -> Path.Set.t t - -(** Like [paths_matching], but don't return the resulting set. The action - dependency is still registered. *) -val paths_matching_unit : loc:Loc.t -> File_selector.t -> unit t - -(** [paths_existing paths] will require as dependencies the files that actually - exist. *) -val paths_existing : Path.t list -> unit t - -(** [env_var v] records [v] as an environment variable that is read by the - action produced by the action builder. *) -val env_var : string -> unit t - -module Alias_status : sig - type t = - | Defined - | Not_defined - - include Monoid.S with type t := t +val return : 'a -> 'a t +val bind : 'a t -> f:('a -> 'b t) -> 'b t +val map : 'a t -> f:('a -> 'b) -> 'b t +val map2 : 'a t -> 'b t -> f:('a -> 'b -> 'c) -> 'c t +val both : 'a t -> 'b t -> ('a * 'b) t +val all : 'a t list -> 'a list t +val all_unit : unit t list -> unit t + +module List : Monad.List with type 'a t := 'a t + +(** [memoize ?cutoff name t] is an action builder that behaves like [t] except + that its result is computed only once. + + If the caller provides the [cutoff] equality check, we will use it to check + if the result of the computation has changed. If it didn't, we will be able + to skip the recomputation of values that depend on it. *) +val memoize : ?cutoff:('a -> 'a -> bool) -> string -> 'a t -> 'a t + +type ('input, 'output) memo + +(** Same as [Memo.create] but for [Action_builder] *) +val create_memo + : string + -> input:(module Memo.Input with type t = 'i) + -> ?cutoff:('o -> 'o -> bool) + -> ?human_readable_description:('i -> User_message.Style.t Pp.t) + -> ('i -> 'o t) + -> ('i, 'o) memo + +(** Same as [Memo.exec] but for [Action_builder]'s memos *) +val exec_memo : ('i, 'o) memo -> 'i -> 'o t + +(** [goal t] ignores all facts that have been accumulated about the dependencies + of [t]. For example, [goal (path p)] declares that a path [p] contributes to + the "goal" of the resulting action builder, which means [p] must be built, + but the contents of [p] is irrelevant. *) +val goal : 'a t -> 'a t + +(** An action builder with no dependencies. Consider passing [Memo.of_thunk] to delay + forcing the computation until the action's dependencies need to be determined. + + If you're thinking of using [Process.run] here, check that: (i) you don't in + fact need [Command.run], and that (ii) [Process.run] only reads the declared + build rule dependencies. *) +val of_memo : 'a Memo.t -> 'a t + +(** Record the given set as dependencies of the action produced by the action builder. *) +val record : 'a -> Dep.Set.t -> f:(Dep.t -> Dep.Fact.t Memo.t) -> 'a t + +(** Record a given Memo computation as a "dependency" of the action builder, i.e., require + that it must succeed. Consider passing [Memo.of_thunk] to delay forcing the computation + until the action's dependencies need to be determined. *) +val record_success : unit Memo.t -> unit t + +module Expert : sig + (** Like [record] but records a dependency on a *source* file. Evaluating the resulting + [t] in the [Eager] mode will raise a user error if the file can't be digested. + + This function is in the [Expert] module because depending on files in the source + directory is usually a mistake. As of 2023-11-14, we use this function only for + setting up the rules that copy files from the source to the build directory. *) + val record_dep_on_source_file_exn + : 'a + -> ?loc:(unit -> Loc.t option Memo.t) + -> Path.Source.t + -> 'a t end -val alias : Alias.t -> unit t +(** {1 Evaluation} *) -module Alias_build_info : sig - (** When looking up aliases, [alias_status] represents whether the alias was - defined in that directory or not. - - For custom traversals, [allowed_build_only_subdirs] provides additional - information about immediate, build-only (not present in source) - sub-directories that the traversal might want to look under. *) - type t = - { alias_status : Alias_status.t - ; allowed_build_only_subdirs : Filename.Set.t - } -end - -module Alias_rec (_ : sig - (* This API isn't fully baked yet. We might move it to the rules *) - - (** [traverse dir ~f] traverses [dir] and evaluates [f] for every directory. - Returns [Defined] if [f] returned [Defined] at least once. [Not_defined] - otherwise. *) - val traverse - : Path.Build.t - -> f:(path:Path.Build.t -> Alias_build_info.t t) - -> Alias_status.t t - end) : sig - (** Depend on an alias recursively. Return [Defined] if the alias is defined - in at least one directory, and [Not_defined] otherwise. *) - val dep_on_alias_rec : Alias.Name.t -> Path.Build.t -> Alias_status.t t -end +(** Evaluate a [t] and collect the set of its dependencies. This avoids doing the build + work required for finding the facts about those dependencies, so you should use this + function if you don't need the facts. *) +val evaluate_and_collect_deps : 'a t -> ('a * Dep.Set.t) Memo.t -(** [dyn_memo_deps m] adds the dependencies computed by [m] while returning the - extra value. *) -val dyn_memo_deps : (Dep.Set.t * 'a) Memo.t -> 'a t - -(** Record dynamic dependencies *) -val dyn_paths : ('a * Path.t list) t -> 'a t - -val dyn_paths_unit : Path.t list t -> unit t -val dyn_path_set : ('a * Path.Set.t) t -> 'a t -val dyn_path_set_reuse : Path.Set.t t -> Path.Set.t t - -(** [contents path] returns a description that when run will return the contents - of the file at [path]. *) -val contents : Path.t -> string t - -(** [lines_of path] returns a description that when run will return the contents - of the file at [path] as a list of lines. *) -val lines_of : Path.t -> string list t - -(** Load an S-expression from a file *) -val read_sexp : Path.t -> Dune_sexp.Ast.t t - -(** Evaluates to [true] if the file is present on the file system or is the - target of a rule. It doesn't add the path as dependency *) -val file_exists : Path.t -> bool t - -(** [if_file_exists p ~then ~else] is a description that behaves like [then_] if - [file_exists p] evaluates to [true], and [else_] otherwise. *) -val if_file_exists : Path.t -> then_:'a t -> else_:'a t -> 'a t - -(** Create a file with the given contents. *) -val write_file - : ?perm:Action.File_perm.t - -> Path.Build.t - -> string - -> Action.Full.t With_targets.t - -val write_file_dyn - : ?perm:Action.File_perm.t - -> Path.Build.t - -> string t - -> Action.Full.t With_targets.t - -val with_stdout_to - : ?perm:Action.File_perm.t - -> Path.Build.t - -> Action.Full.t t - -> Action.Full.t With_targets.t - -val copy : src:Path.t -> dst:Path.Build.t -> Action.Full.t With_targets.t -val symlink : src:Path.t -> dst:Path.Build.t -> Action.Full.t With_targets.t -val symlink_dir : src:Path.t -> dst:Path.Build.t -> Action.Full.t With_targets.t -val create_file : ?perm:Action.File_perm.t -> Path.Build.t -> Action.Full.t With_targets.t - -(** Merge a list of actions accumulating the sets of their targets. *) -val progn : Action.Full.t With_targets.t list -> Action.Full.t With_targets.t - -(** A version of [dyn_of_memo] that makes it convenient to declare dynamic - action dependencies. *) -val dyn_of_memo_deps : ('a * Dep.Set.t) Memo.t t -> 'a t +(** Evaluate a [t] and collect the set of its dependencies along with facts about them. + Note that finding [t]'s facts requires building all of [t]'s dependencies. *) +val evaluate_and_collect_facts : 'a t -> ('a * Dep.Facts.t) Memo.t diff --git a/src/dune_engine/action_builder0.ml b/src/dune_engine/action_builder0.ml deleted file mode 100644 index a34ef62e67b..00000000000 --- a/src/dune_engine/action_builder0.ml +++ /dev/null @@ -1,251 +0,0 @@ -open Import - -type 'a eval_mode = - | Lazy : unit eval_mode - | Eager : Dep.Fact.t eval_mode - -type 'a thunk = { f : 'm. 'm eval_mode -> ('a * 'm Dep.Map.t) Memo.t } [@@unboxed] - -module Deps_or_facts = struct - let union : type a. a eval_mode -> a Dep.Map.t -> a Dep.Map.t -> a Dep.Map.t = - fun mode a b -> - match mode with - | Lazy -> Dep.Set.union a b - | Eager -> Dep.Facts.union a b - ;; - - let union_all mode l = List.fold_left l ~init:Dep.Map.empty ~f:(union mode) -end - -module T = struct - open Memo.O - - module M = struct - type 'a t = 'a thunk - - let return x = { f = (fun _mode -> Memo.return (x, Dep.Map.empty)) } - - let map t ~f = - { f = - (fun mode -> - let+ x, deps = t.f mode in - f x, deps) - } - ;; - - let bind t ~f = - { f = - (fun mode -> - let* x, deps1 = t.f mode in - let+ y, deps2 = (f x).f mode in - y, Deps_or_facts.union mode deps1 deps2) - } - ;; - end - - include M - - let both x y = - { f = - (fun mode -> - let+ x, deps1 = x.f mode - and+ y, deps2 = y.f mode in - (x, y), Deps_or_facts.union mode deps1 deps2) - } - ;; - - let all xs = - { f = - (fun mode -> - let+ res = Memo.parallel_map xs ~f:(fun x -> x.f mode) in - let res, facts = List.split res in - res, Deps_or_facts.union_all mode facts) - } - ;; - - let of_memo m = - { f = - (fun _mode -> - let+ x = m in - x, Dep.Map.empty) - } - ;; - - module O = struct - let ( >>> ) a b = - { f = - (fun mode -> - let+ ((), deps_a), (b, deps_b) = - Memo.fork_and_join (fun () -> a.f mode) (fun () -> b.f mode) - in - b, Deps_or_facts.union mode deps_a deps_b) - } - ;; - - let ( >>= ) t f = bind t ~f - let ( >>| ) t f = map t ~f - let ( and+ ) = both - let ( and* ) = both - let ( let+ ) t f = map t ~f - let ( let* ) t f = bind t ~f - end - - module List = struct - include Monad.List (struct - include M - include Monad.Make (M) - end) - - let map l ~f = - { f = - (fun mode -> - let+ res = Memo.parallel_map l ~f:(fun x -> (f x).f mode) in - let res, deps = List.split res in - res, Deps_or_facts.union_all mode deps) - } - ;; - - let concat_map l ~f = - { f = - (fun mode -> - let+ res = Memo.parallel_map l ~f:(fun x -> (f x).f mode) in - let res, deps = List.split res in - List.concat res, Deps_or_facts.union_all mode deps) - } - ;; - end -end - -include T -open O - -open struct - module List = Stdune.List -end - -let of_thunk t = t -let run t mode = t.f mode - -let force_lazy_or_eager - : type a b. - a eval_mode - -> (b * Dep.Set.t) Memo.Lazy.t Lazy.t - -> (b * Dep.Facts.t) Memo.Lazy.t - -> (b * a Dep.Map.t) Memo.t - = - fun mode lazy_ eager -> - match mode with - | Lazy -> Memo.Lazy.force (Lazy.force lazy_) - | Eager -> Memo.Lazy.force eager -;; - -let memoize ?cutoff name t = - let lazy_ : ('a * Dep.Set.t) Memo.Lazy.t Lazy.t = - lazy - (let cutoff = - Option.map cutoff ~f:(fun equal -> Tuple.T2.equal equal Dep.Set.equal) - in - Memo.lazy_ ?cutoff ~name:(name ^ "(lazy)") (fun () -> t.f Lazy)) - in - (* Unlike [lazy_], [eager] doesn't have the outer [Lazy.t] wrapper because most [Eager] - nodes end up getting forced during every build. *) - let eager : ('a * Dep.Facts.t) Memo.Lazy.t = - let cutoff = - Option.map cutoff ~f:(fun equal -> Tuple.T2.equal equal Dep.Facts.equal) - in - Memo.lazy_ ?cutoff ~name (fun () -> t.f Eager) - in - { f = (fun mode -> force_lazy_or_eager mode lazy_ eager) } -;; - -let ignore x = map x ~f:ignore - -let map2 x y ~f = - let+ x = x - and+ y = y in - f x y -;; - -let push_stack_frame ~human_readable_description f = - { f = - (fun mode -> - Memo.push_stack_frame ~human_readable_description (fun () -> (f ()).f mode)) - } -;; - -let delayed f = - let+ () = return () in - f () -;; - -let all_unit (xs : unit t list) = - { f = - (fun mode -> - let open Memo.O in - let+ res = Memo.parallel_map xs ~f:(fun x -> x.f mode) in - let deps = List.map res ~f:snd in - (), Deps_or_facts.union_all mode deps) - } -;; - -type fail = { fail : 'a. unit -> 'a } - -let fail x = - let+ () = return () in - x.fail () -;; - -type ('input, 'output) memo = - { lazy_ : ('input, 'output * Dep.Set.t) Memo.Table.t Lazy.t - ; eager : ('input, 'output * Dep.Facts.t) Memo.Table.t Lazy.t - } - -let create_memo name ~input ?cutoff ?human_readable_description f = - let lazy_ = - lazy - (let cutoff = - Option.map cutoff ~f:(fun f (a, deps1) (b, deps2) -> - f a b && Dep.Set.equal deps1 deps2) - in - let name = name ^ "(lazy)" in - Memo.create name ~input ?cutoff ?human_readable_description (fun x -> (f x).f Lazy)) - and eager = - lazy - (let cutoff = - Option.map cutoff ~f:(fun f (a, facts1) (b, facts2) -> - f a b && Dep.Map.equal facts1 facts2 ~equal:Dep.Fact.equal) - in - Memo.create name ~input ?cutoff ?human_readable_description (fun x -> - (f x).f Eager)) - in - { lazy_; eager } -;; - -let exec_memo : type i o m. (i, o) memo -> i -> m eval_mode -> (o * m Dep.Map.t) Memo.t = - fun memo i mode -> - match mode with - | Lazy -> Memo.exec (Lazy.force memo.lazy_) i - | Eager -> Memo.exec (Lazy.force memo.eager) i -;; - -let exec_memo m i = { f = (fun mode -> exec_memo m i mode) } - -let goal t = - { f = - (fun mode -> - let open Memo.O in - let+ a, (_irrelevant_for_goals : _ Dep.Map.t) = t.f mode in - a, Dep.Map.empty) - } -;; - -let of_memo_join f = - { f = - (fun mode -> - let open Memo.O in - let* t = f in - t.f mode) - } -;; - -let dyn_of_memo f = f >>= of_memo diff --git a/src/dune_engine/action_builder0.mli b/src/dune_engine/action_builder0.mli deleted file mode 100644 index 0a34e41d2b9..00000000000 --- a/src/dune_engine/action_builder0.mli +++ /dev/null @@ -1,108 +0,0 @@ -open Import - -type 'a t - -module O : sig - val ( >>> ) : unit t -> 'a t -> 'a t - val ( >>= ) : 'a t -> ('a -> 'b t) -> 'b t - val ( >>| ) : 'a t -> ('a -> 'b) -> 'b t - val ( let* ) : 'a t -> ('a -> 'b t) -> 'b t - val ( and* ) : 'a t -> 'b t -> ('a * 'b) t - val ( let+ ) : 'a t -> ('a -> 'b) -> 'b t - val ( and+ ) : 'a t -> 'b t -> ('a * 'b) t -end - -val return : 'a -> 'a t -val bind : 'a t -> f:('a -> 'b t) -> 'b t -val map : 'a t -> f:('a -> 'b) -> 'b t -val map2 : 'a t -> 'b t -> f:('a -> 'b -> 'c) -> 'c t -val both : 'a t -> 'b t -> ('a * 'b) t -val ignore : 'a t -> unit t -val all : 'a t list -> 'a list t -val all_unit : unit t list -> unit t - -module List : Monad.List with type 'a t := 'a t - -val push_stack_frame - : human_readable_description:(unit -> User_message.Style.t Pp.t) - -> (unit -> 'a t) - -> 'a t - -(** Delay a static computation until the description is evaluated *) -val delayed : (unit -> 'a) -> 'a t - -type fail = { fail : 'a. unit -> 'a } - -(** Always fail when executed. We pass a function rather than an exception to - get a proper backtrace *) -val fail : fail -> _ t - -(** [memoize ?cutoff name t] is an action builder that behaves like [t] except - that its result is computed only once. - - If the caller provides the [cutoff] equality check, we will use it to check - if the result of the computation has changed. If it didn't, we will be able - to skip the recomputation of values that depend on it. *) -val memoize : ?cutoff:('a -> 'a -> bool) -> string -> 'a t -> 'a t - -type ('input, 'output) memo - -(** Same as [Memo.create] but for [Action_builder] *) -val create_memo - : string - -> input:(module Memo.Input with type t = 'i) - -> ?cutoff:('o -> 'o -> bool) - -> ?human_readable_description:('i -> User_message.Style.t Pp.t) - -> ('i -> 'o t) - -> ('i, 'o) memo - -(** Same as [Memo.exec] but for [Action_builder]'s memos *) -val exec_memo : ('i, 'o) memo -> 'i -> 'o t - -(** [goal t] ignores all facts that have been accumulated about the dependencies - of [t]. For example, [goal (path p)] declares that a path [p] contributes to - the "goal" of the resulting action builder, which means [p] must be built, - but the contents of [p] is irrelevant. *) -val goal : 'a t -> 'a t - -(** If you're thinking of using [Process.run] here, check that: (i) you don't in - fact need [Command.run], and that (ii) [Process.run] only reads the declared - build rule dependencies. *) -val of_memo : 'a Memo.t -> 'a t - -(** Like [of_memo] but collapses the two levels of [t]. *) -val of_memo_join : 'a t Memo.t -> 'a t - -(** If you're thinking of using [Process.run] here, check that: (i) you don't in - fact need [Command.run], and that (ii) [Process.run] only reads the declared - build rule dependencies. *) -val dyn_of_memo : 'a Memo.t t -> 'a t - -(** {1 Execution} *) - -(** Evaluation mode. - - In [Lazy] mode, dependencies are only collected. In [Eager] mode, - dependencies are build as soon as they are recorded and their facts are - returned. - - If you want to both evaluate an action builder and build the collected - dependencies, using [Eager] mode will increase parallelism. If you only want - to know the set of dependencies, using [Lazy] will avoid unnecessary work. *) -type 'a eval_mode = - | Lazy : unit eval_mode - | Eager : Dep.Fact.t eval_mode - -(** Execute an action builder. *) -val run : 'a t -> 'b eval_mode -> ('a * 'b Dep.Map.t) Memo.t - -(** {1 Low-level} *) - -type 'a thunk = { f : 'm. 'm eval_mode -> ('a * 'm Dep.Map.t) Memo.t } [@@unboxed] - -val of_thunk : 'a thunk -> 'a t - -module Deps_or_facts : sig - val union : 'a eval_mode -> 'a Dep.Map.t -> 'a Dep.Map.t -> 'a Dep.Map.t - val union_all : 'a eval_mode -> 'a Dep.Map.t list -> 'a Dep.Map.t -end diff --git a/src/dune_engine/build_config.ml b/src/dune_engine/build_config.ml index 1ab1c2b3a66..e6e21df7180 100644 --- a/src/dune_engine/build_config.ml +++ b/src/dune_engine/build_config.ml @@ -1,5 +1,4 @@ open Import -module Action_builder = Action_builder0 module Gen_rules = struct module Context_type = Build_config_intf.Context_type diff --git a/src/dune_engine/build_config_intf.ml b/src/dune_engine/build_config_intf.ml index d94763c8817..333f04b2fcd 100644 --- a/src/dune_engine/build_config_intf.ml +++ b/src/dune_engine/build_config_intf.ml @@ -1,5 +1,4 @@ open Import -module Action_builder = Action_builder0 module Context_type = struct type t = diff --git a/src/dune_engine/build_system.ml b/src/dune_engine/build_system.ml index d3158e2d075..4e8093297fb 100644 --- a/src/dune_engine/build_system.ml +++ b/src/dune_engine/build_system.ml @@ -1,6 +1,5 @@ open Import open Memo.O -module Action_builder = Action_builder0 module Error = Build_system_error module Progress = struct @@ -132,7 +131,7 @@ module type Rec = sig val build_file : Path.t -> Digest.t Memo.t val build_dir : Path.t -> (Digest.t * Digest.t Path.Build.Map.t) Memo.t - val build_deps : Dep.Set.t -> Dep.Facts.t Memo.t + val build_dep : Dep.t -> Dep.Fact.t Memo.t val eval_deps : 'a Action_builder.eval_mode -> Dep.Set.t -> 'a Dep.Map.t Memo.t val execute_rule : Rule.t -> rule_execution_result Memo.t @@ -199,15 +198,9 @@ end = struct Memo.return Dep.Fact.nothing ;; + (* As of 2023-11-14, this function is used only for actions with dynamic dependencies. *) let build_deps deps = Dep.Map.parallel_map deps ~f:(fun dep () -> build_dep dep) - let eval_deps : type a. a Action_builder.eval_mode -> Dep.Set.t -> a Dep.Map.t Memo.t = - fun mode deps -> - match mode with - | Lazy -> Memo.return deps - | Eager -> build_deps deps - ;; - let select_sandbox_mode (config : Sandbox_config.t) ~loc ~sandboxing_preference = (* Rules with (mode patch-back-source-tree) are special and are not affected by sandboxing preferences. *) @@ -388,7 +381,6 @@ end = struct Fiber.return ()) (fun () -> with_locks locks ~f:(fun () -> - let build_deps deps = Memo.run (build_deps deps) in let* action_exec_result = let input = { Action_exec.root @@ -401,7 +393,9 @@ end = struct } in match (Build_config.get ()).action_runner input with - | None -> Action_exec.exec input ~build_deps + | None -> + let build_deps deps = Memo.run (build_deps deps) in + Action_exec.exec input ~build_deps | Some runner -> Action_runner.exec_action runner input in let* action_exec_result = Action_exec.Exec_result.ok_exn action_exec_result in @@ -450,7 +444,7 @@ end = struct function [(Build_config.get ()).execution_parameters] is likely memoized, and the result is not expected to change often, so we do not sacrifice too much performance here by executing it sequentially. *) - let* action, deps = Action_builder.run action Eager in + let* action, deps = Action_builder.evaluate_and_collect_facts action in let wrap_fiber f = Memo.of_reproducible_fiber (if Loc.is_none loc @@ -653,12 +647,7 @@ end = struct ~info:(if Loc.is_none loc then Internal else From_dune_file loc) ~targets:(Targets.File.create target) ~mode:Standard - (Action_builder.of_thunk - { f = - (fun mode -> - let+ deps = eval_deps mode deps in - act.action, deps) - }) + (Action_builder.record act.action deps ~f:build_dep) in let+ { deps = _; targets = _ } = execute_rule_impl @@ -837,21 +826,16 @@ end = struct ])) ;; - let dep_on_anonymous_action (x : Rule.Anonymous_action.t Action_builder.t) - : _ Action_builder.t - = - Action_builder.of_thunk - { f = - (fun (type m) (mode : m Action_builder.eval_mode) -> - match mode with - | Lazy -> Memo.return ((), Dep.Map.empty) - | Eager -> - let* action, facts = Action_builder.run x Eager in - let+ () = execute_action action ~observing_facts:facts in - (), Dep.Map.empty) - } + let execute_anonymous_action action = + let* action, facts = Action_builder.evaluate_and_collect_facts action in + execute_action action ~observing_facts:facts ;; + let dep_on_anonymous_action (action : Rule.Anonymous_action.t Action_builder.t) + : unit Action_builder.t + = + Action_builder.record_success (Memo.of_thunk_apply execute_anonymous_action action) + let dep_on_alias_definition (definition : Rules.Dir_rules.Alias_spec.item) = match definition with | Deps x -> x @@ -864,7 +848,8 @@ end = struct >>= Memo.parallel_map ~f:(fun (loc, definition) -> Memo.push_stack_frame (fun () -> - Action_builder.run (dep_on_alias_definition definition) Eager >>| snd) + Action_builder.evaluate_and_collect_facts (dep_on_alias_definition definition) + >>| snd) ~human_readable_description:(fun () -> Alias.describe alias ~loc)) in Dep.Facts.group_paths_as_fact_files l @@ -1154,10 +1139,21 @@ let build_file p = () ;; -let read_file p ~f = +let with_file p ~f = let+ () = build_file p in f p ;; +(* CR-someday amokhov: Try running [Io.read_file] in a separate thread. *) +let read_file = + Memo.exec + (Memo.create_with_store + "Build_system.read_file" + ~store:(module Path.Table) + ~input:(module Path) + ~cutoff:String.equal + (fun path -> with_file path ~f:Io.read_file)) +;; + let state = State.t let errors = State.errors diff --git a/src/dune_engine/build_system.mli b/src/dune_engine/build_system.mli index 1a3dc81a0b4..9bdb7131806 100644 --- a/src/dune_engine/build_system.mli +++ b/src/dune_engine/build_system.mli @@ -1,19 +1,22 @@ (** The core of the build system *) open Import -module Action_builder := Action_builder0 (** Build a file. *) val build_file : Path.t -> unit Memo.t -(** Build a file and access its contents with [f]. *) -val read_file : Path.t -> f:(Path.t -> 'a) -> 'a Memo.t +(** Build a file and read its contents with [f]. The execution of [f] is not memoized, so + call sites should be careful to avoid duplicating [f]'s work. *) +val with_file : Path.t -> f:(Path.t -> 'a) -> 'a Memo.t + +(** Build a file and read its contents. Like [with_file ~f:Io.read_file] but memoized. *) +val read_file : Path.t -> string Memo.t (** Return [true] if a file exists or is buildable *) val file_exists : Path.t -> bool Memo.t -(** Build a set of dependencies and return learned facts about them. *) -val build_deps : Dep.Set.t -> Dep.Facts.t Memo.t +(** Record the given set as dependencies of the action produced by the action builder. *) +val record_deps : Dep.Set.t -> unit Action_builder.t (** [eval_pred glob] returns the list of files in [File_selector.dir glob] that matches [File_selector.predicate glob]. The list of files includes the list diff --git a/src/dune_engine/dep.ml b/src/dune_engine/dep.ml index 667284b785b..7789816ef6f 100644 --- a/src/dune_engine/dep.ml +++ b/src/dune_engine/dep.ml @@ -210,11 +210,51 @@ module Fact = struct let alias _alias files = Alias files end +module Set = struct + module M = Set.Of_map (T) (Map) + include M + + let of_files l = of_list_map l ~f:file + let of_files_set = Path.Set.fold ~init:empty ~f:(fun f acc -> add acc (file f)) + let add_paths t paths = Path.Set.fold paths ~init:t ~f:(fun p set -> add set (File p)) + let encode t = Dune_sexp.Encoder.list encode (to_list t) + + (* This is to force the rules to be loaded for directories without files when + depending on [(source_tree x)]. Otherwise, we wouldn't clean up stale + directories in directories that contain no file. *) + let dir_without_files_dep dir = + file_selector (File_selector.of_predicate_lang ~dir Predicate_lang.false_) + ;; + + let of_source_files ~files ~empty_directories = + let init = Path.Set.fold files ~init:empty ~f:(fun path acc -> add acc (file path)) in + Path.Set.fold empty_directories ~init ~f:(fun path acc -> + add acc (dir_without_files_dep path)) + ;; + + let digest t = + fold t ~init:[] ~f:(fun dep acc : Stable_for_digest.t list -> + match dep with + | Env var -> Env var :: acc + | Universe -> Universe :: acc + | File p -> File (Path.to_string p) :: acc + | File_selector fs -> File_selector (File_selector.to_dyn fs) :: acc + | Alias a -> + Alias + { dir = Path.Build.to_string (Alias.dir a) + ; name = Alias.Name.to_string (Alias.name a) + } + :: acc) + |> Digest.generic + ;; +end + module Facts = struct type t = Fact.t Map.t let equal x y = Map.equal ~equal:Fact.equal x y let empty = Map.empty + let record_facts set ~f = Map.parallel_map set ~f:(fun dep () -> f dep) let union a b = Map.union a b ~f:(fun _ a b -> @@ -290,42 +330,3 @@ module Facts = struct Digest.generic facts ;; end - -module Set = struct - module M = Set.Of_map (T) (Map) - include M - - let of_files l = of_list_map l ~f:file - let of_files_set = Path.Set.fold ~init:empty ~f:(fun f acc -> add acc (file f)) - let add_paths t paths = Path.Set.fold paths ~init:t ~f:(fun p set -> add set (File p)) - let encode t = Dune_sexp.Encoder.list encode (to_list t) - - (* This is to force the rules to be loaded for directories without files when - depending on [(source_tree x)]. Otherwise, we wouldn't clean up stale - directories in directories that contain no file. *) - let dir_without_files_dep dir = - file_selector (File_selector.of_predicate_lang ~dir Predicate_lang.false_) - ;; - - let of_source_files ~files ~empty_directories = - let init = Path.Set.fold files ~init:empty ~f:(fun path acc -> add acc (file path)) in - Path.Set.fold empty_directories ~init ~f:(fun path acc -> - add acc (dir_without_files_dep path)) - ;; - - let digest t = - fold t ~init:[] ~f:(fun dep acc : Stable_for_digest.t list -> - match dep with - | Env var -> Env var :: acc - | Universe -> Universe :: acc - | File p -> File (Path.to_string p) :: acc - | File_selector fs -> File_selector (File_selector.to_dyn fs) :: acc - | Alias a -> - Alias - { dir = Path.Build.to_string (Alias.dir a) - ; name = Alias.Name.to_string (Alias.name a) - } - :: acc) - |> Digest.generic - ;; -end diff --git a/src/dune_engine/dep.mli b/src/dune_engine/dep.mli index 781462d8a4a..7bd66782913 100644 --- a/src/dune_engine/dep.mli +++ b/src/dune_engine/dep.mli @@ -63,7 +63,27 @@ module Fact : sig val file_selector : File_selector.t -> Files.t -> t end +module Set : sig + include Set.S with type elt = t and type 'a map := 'a Map.t and type t = unit Map.t + + (** [of_source_files ~files ~empty_directories] depend on all source files + [files]. + + Dependency on a [files] requires special care for empty directories. Empty + directories need to be loaded so that we clean up stale artifacts in such + directories. This is why [empty_directories] must be provided *) + val of_source_files : files:Path.Set.t -> empty_directories:Path.Set.t -> t + + val of_files : Path.t list -> t + val of_files_set : Path.Set.t -> t + val encode : t -> Dune_sexp.t + val add_paths : t -> Path.Set.t -> t + val digest : t -> Digest.t +end + module Facts : sig + type dep := t + (* There is an invariant that is not currently enforced: values correspond to keys. For example, we can't have [Map.find (File f) = File_selector _]. *) type t = Fact.t Map.t @@ -72,6 +92,7 @@ module Facts : sig val empty : t val union : t -> t -> t val union_all : t list -> t + val record_facts : Set.t -> f:(dep -> Fact.t Memo.t) -> t Memo.t (** Return all file paths, expanding aliases. *) val paths : t -> Digest.t Path.Map.t @@ -90,21 +111,3 @@ module Facts : sig val digest : t -> env:Env.t -> Digest.t val to_dyn : t -> Dyn.t end - -module Set : sig - include Set.S with type elt = t and type 'a map := 'a Map.t and type t = unit Map.t - - (** [of_source_files ~files ~empty_directories] depend on all source files - [files]. - - Dependency on a [files] requires special care for empty directories. Empty - directories need to be loaded so that we clean up stale artifacts in such - directories. This is why [empty_directories] must be provided *) - val of_source_files : files:Path.Set.t -> empty_directories:Path.Set.t -> t - - val of_files : Path.t list -> t - val of_files_set : Path.Set.t -> t - val encode : t -> Dune_sexp.t - val add_paths : t -> Path.Set.t -> t - val digest : t -> Digest.t -end diff --git a/src/dune_engine/fs_memo.ml b/src/dune_engine/fs_memo.ml index 6e265dd51f5..9d5d5333b9e 100644 --- a/src/dune_engine/fs_memo.ml +++ b/src/dune_engine/fs_memo.ml @@ -302,6 +302,36 @@ let file_digest ?(force_update = false) path = Fs_cache.read Fs_cache.Untracked.file_digest path ;; +let file_digest_exn ?loc path = + let report_user_error details = + let+ loc = + match loc with + | None -> Memo.return None + | Some loc -> loc () + in + User_error.raise + ?loc + ([ Pp.textf + "File unavailable: %s" + (Path.Outside_build_dir.to_string_maybe_quoted path) + ] + @ details) + in + file_digest path + >>= function + | Ok digest -> Memo.return digest + | Error No_such_file -> report_user_error [] + | Error Broken_symlink -> report_user_error [ Pp.text "Broken symbolic link" ] + | Error Cyclic_symlink -> report_user_error [ Pp.text "Cyclic symbolic link" ] + | Error (Unexpected_kind st_kind) -> + report_user_error + [ Pp.textf "This is not a regular file (%s)" (File_kind.to_string st_kind) ] + | Error (Unix_error unix_error) -> + report_user_error [ Unix_error.Detailed.pp ~prefix:"Reason: " unix_error ] + | Error (Unrecognized exn) -> + report_user_error [ Pp.textf "%s" (Printexc.to_string exn) ] +;; + let dir_contents ?(force_update = false) path = if force_update then Fs_cache.evict Fs_cache.Untracked.dir_contents path; let+ () = Watcher.watch ~try_to_watch_via_parent:false path in diff --git a/src/dune_engine/fs_memo.mli b/src/dune_engine/fs_memo.mli index dc8a607c914..ef412616cc4 100644 --- a/src/dune_engine/fs_memo.mli +++ b/src/dune_engine/fs_memo.mli @@ -34,6 +34,12 @@ val file_digest -> Path.Outside_build_dir.t -> Cached_digest.Digest_result.t Memo.t +(** Like [file_digest] but raises a user error if the resulting digest is not [Ok _]. *) +val file_digest_exn + : ?loc:(unit -> Loc.t option Memo.t) + -> Path.Outside_build_dir.t + -> Digest.t Memo.t + (** Like [Io.Untracked.with_lexbuf_from_file] but declares a dependency on the path. *) val with_lexbuf_from_file diff --git a/src/dune_engine/load_rules.ml b/src/dune_engine/load_rules.ml index e7974e22385..9e2195833c8 100644 --- a/src/dune_engine/load_rules.ml +++ b/src/dune_engine/load_rules.ml @@ -1,6 +1,5 @@ open Import open Memo.O -module Action_builder = Action_builder0 module Gen_rules = Build_config.Gen_rules module Context_type = Gen_rules.Context_type module Build_only_sub_dirs = Gen_rules.Build_only_sub_dirs @@ -269,41 +268,6 @@ let no_rule_found ~loc fn = [ "fn", Path.Build.to_dyn fn ] ;; -let source_or_external_file_digest path = - let report_user_error details = - let+ loc = Current_rule_loc.get () in - User_error.raise - ?loc - ([ Pp.textf - "File unavailable: %s" - (Path.Outside_build_dir.to_string_maybe_quoted path) - ] - @ details) - in - Fs_memo.file_digest path - >>= function - | Ok digest -> Memo.return digest - | Error No_such_file -> report_user_error [] - | Error Broken_symlink -> report_user_error [ Pp.text "Broken symbolic link" ] - | Error Cyclic_symlink -> report_user_error [ Pp.text "Cyclic symbolic link" ] - | Error (Unexpected_kind st_kind) -> - report_user_error - [ Pp.textf "This is not a regular file (%s)" (File_kind.to_string st_kind) ] - | Error (Unix_error unix_error) -> - report_user_error [ Unix_error.Detailed.pp ~prefix:"Reason: " unix_error ] - | Error (Unrecognized exn) -> - report_user_error [ Pp.textf "%s" (Printexc.to_string exn) ] -;; - -let eval_source_file : type a. a Action_builder.eval_mode -> Path.Source.t -> a Memo.t = - fun mode path -> - match mode with - | Lazy -> Memo.return () - | Eager -> - let+ d = source_or_external_file_digest (In_source_dir path) in - Dep.Fact.file (Path.source path) d -;; - module rec Load_rules : sig val load_dir : dir:Path.t -> Loaded.t Memo.t val is_under_directory_target : Path.t -> bool Memo.t @@ -314,27 +278,29 @@ module rec Load_rules : sig end = struct open Load_rules - let create_copy_rules ~ctx_dir ~non_target_source_files = - Path.Source.Set.to_list_map non_target_source_files ~f:(fun path -> - let ctx_path = Path.Build.append_source ctx_dir path in - let build = - Action_builder.of_thunk - { f = - (fun mode -> - let+ fact = eval_source_file mode path in - let path = Path.source path in - ( Action.Full.make - (Action.copy path ctx_path) - (* There's an [assert false] in [prepare_managed_paths] - that blows up if we try to sandbox this. *) - ~sandbox:Sandbox_config.no_sandboxing - , Dep.Map.singleton (Dep.file path) fact )) - } - in + let copy_source_action ~src_path ~build_path : Action.Full.t Action_builder.t = + let action = + Action.Full.make + (Action.copy (Path.source src_path) build_path) + (* Sandboxing this action doesn't make much sense: if we can copy [src_path] to + the sandbox, we might as well copy it to the build directory directly. *) + ~sandbox:Sandbox_config.no_sandboxing + in + Action_builder.Expert.record_dep_on_source_file_exn + action + ~loc:Current_rule_loc.get + src_path + ;; + + let create_copy_rules ~dir ~ctx_dir ~non_target_source_filenames = + Filename.Set.to_list_map non_target_source_filenames ~f:(fun filename -> + let src_path = Path.Source.relative dir filename in + let build_path = Path.Build.append_source ctx_dir src_path in Rule.make - ~info:(Source_file_copy path) - ~targets:(Targets.File.create ctx_path) - build) + ~context:None + ~info:(Source_file_copy src_path) + ~targets:(Targets.File.create build_path) + (copy_source_action ~src_path ~build_path)) ;; let compile_rules ~dir ~source_dirs rules = @@ -988,8 +954,8 @@ type rule_or_source = let get_rule_or_source path = match Path.destruct_build_dir path with | `Outside path -> - let+ d = source_or_external_file_digest path in - Source d + let+ digest = Fs_memo.file_digest_exn ~loc:Current_rule_loc.get path in + Source digest | `Inside path -> get_rule_internal path >>= (function diff --git a/src/dune_engine/load_rules.mli b/src/dune_engine/load_rules.mli index 82bb1074e8e..8ddbdf39482 100644 --- a/src/dune_engine/load_rules.mli +++ b/src/dune_engine/load_rules.mli @@ -1,7 +1,6 @@ (** Loading build rules *) open Import -module Action_builder := Action_builder0 (** A way to determine the [Loc.t] of the current rule. Set by [Build_system]. *) val set_current_rule_loc : (unit -> Loc.t option Memo.t) -> unit diff --git a/src/dune_engine/reflection.ml b/src/dune_engine/reflection.ml index 14a06fa1b3c..bc315ea2fbe 100644 --- a/src/dune_engine/reflection.ml +++ b/src/dune_engine/reflection.ml @@ -30,9 +30,8 @@ end = struct >>= Memo.parallel_map ~f:(fun (loc, definition) -> Memo.push_stack_frame (fun () -> - Action_builder.run + Action_builder.evaluate_and_collect_deps (Build_system.dep_on_alias_definition definition) - Lazy >>| snd) ~human_readable_description:(fun () -> Alias.describe alias ~loc)) in @@ -59,7 +58,7 @@ let evaluate_rule = "evaluate-rule" ~input:(module Non_evaluated_rule) (fun rule -> - let* action, deps = Action_builder.run rule.action Lazy in + let* action, deps = Action_builder.evaluate_and_collect_deps rule.action in let* expanded_deps = Expand.deps deps in Memo.return { Rule.id = rule.id @@ -84,7 +83,7 @@ let eval ~recursive ~request = | Some rule -> evaluate_rule rule >>| Option.some) >>| List.filter_opt in - let* (), deps = Action_builder.run request Lazy in + let* (), deps = Action_builder.evaluate_and_collect_deps request in let* root_rules = rules_of_deps deps in Rule_top_closure.top_closure root_rules diff --git a/src/dune_engine/rule.ml b/src/dune_engine/rule.ml index 703e4f88b6d..21f428eff46 100644 --- a/src/dune_engine/rule.ml +++ b/src/dune_engine/rule.ml @@ -1,5 +1,4 @@ open Import -module Action_builder = Action_builder0 module Info = struct type t = diff --git a/src/dune_engine/rule.mli b/src/dune_engine/rule.mli index 2f77ea3cb01..25839675f21 100644 --- a/src/dune_engine/rule.mli +++ b/src/dune_engine/rule.mli @@ -1,7 +1,6 @@ (** Representation of rules *) open Import -module Action_builder := Action_builder0 (** Information about the provenance of a build rule. *) module Info : sig diff --git a/src/dune_engine/rules.ml b/src/dune_engine/rules.ml index 99aa08be970..2885f449ef4 100644 --- a/src/dune_engine/rules.ml +++ b/src/dune_engine/rules.ml @@ -1,5 +1,4 @@ open Import -module Action_builder = Action_builder0 module Id = Id.Make () module Dir_rules = struct diff --git a/src/dune_engine/rules.mli b/src/dune_engine/rules.mli index 7c9fec69de2..68fd3998299 100644 --- a/src/dune_engine/rules.mli +++ b/src/dune_engine/rules.mli @@ -1,7 +1,6 @@ (** A collection of rules across a known finite set of directories *) open! Import -module Action_builder := Action_builder0 (** Represent a set of rules producing files in a given directory *) module Dir_rules : sig diff --git a/src/memo/memo.mli b/src/memo/memo.mli index cd717d76ee0..c3f7597861a 100644 --- a/src/memo/memo.mli +++ b/src/memo/memo.mli @@ -57,8 +57,15 @@ val of_reproducible_fiber : 'a Fiber.t -> 'a t therefore be re-executed on every build run. *) val of_non_reproducible_fiber : 'a Fiber.t -> 'a t +(** Convert a thunk to a Memo computation, making sure the thunk runs in the context of + the Memo computation rather than in the current context. + + [of_thunk f] is equivalent to [return () >> f] but is more explicit. *) val of_thunk : (unit -> 'a t) -> 'a t +(** Like [of_thunk] but accepts functions of any argument. *) +val of_thunk_apply : ('a -> 'b t) -> 'a -> 'b t + (** Combine results of two computations executed in sequence. *) val both : 'a t -> 'b t -> ('a * 'b) t diff --git a/vendor/fiber/src/core.ml b/vendor/fiber/src/core.ml index 7cde5272ef6..80152c6c496 100644 --- a/vendor/fiber/src/core.ml +++ b/vendor/fiber/src/core.ml @@ -245,6 +245,7 @@ module Var = struct end let of_thunk f k = f () k +let of_thunk_apply f x k = f x k module O = struct let ( >>> ) a b k = a (fun () -> b k) diff --git a/vendor/fiber/src/fiber.mli b/vendor/fiber/src/fiber.mli index efdfe8a5c92..8fbac56fec5 100644 --- a/vendor/fiber/src/fiber.mli +++ b/vendor/fiber/src/fiber.mli @@ -18,12 +18,15 @@ type 'a fiber := 'a t (** Create a fiber that has already terminated. *) val return : 'a -> 'a t -(** Converts a thunk to a fiber, making sure the thunk runs in the context of - the fiber (rather than applied in the current context). +(** Convert a thunk to a fiber, making sure the thunk runs in the context of the fiber + rather than in the current context. - Equivalent to [(>>=) (return ())], but more explicit. *) + [of_thunk f] is equivalent to [return () >>= f] but is more explicit. *) val of_thunk : (unit -> 'a t) -> 'a t +(** Like [of_thunk] but accepts functions of any argument. *) +val of_thunk_apply : ('a -> 'b t) -> 'a -> 'b t + (** Fiber that never completes. *) val never : 'a t