diff --git a/bin/dune b/bin/dune index 0750a57e864..76593703b4e 100644 --- a/bin/dune +++ b/bin/dune @@ -42,7 +42,8 @@ dune_rpc_private dune_rpc_client spawn - opam_format) + opam_format + xdg) (bootstrap_info bootstrap-info)) ; Installing the dune binary depends on the kind of build: diff --git a/bin/pkg.ml b/bin/pkg.ml index 1cac10934b9..c93b81f06cd 100644 --- a/bin/pkg.ml +++ b/bin/pkg.ml @@ -1,90 +1,116 @@ open Import module Lock_dir = Dune_pkg.Lock_dir +module Fetch = Dune_pkg.Fetch +module Opam = Dune_pkg.Opam +module Repo_selection = Opam.Repo_selection -module Lock = struct - module Repo_selection = struct - module Env = struct - module Source = struct - type t = - | Global - | Pure +module Opam_repository = struct + type t = { url : OpamUrl.t } - let to_string = function - | Global -> "global" - | Pure -> "pure" + let of_url url = { url } - let default = Global + let default = + of_url @@ OpamUrl.of_string "https://opam.ocaml.org/index.tar.gz" - let term = - let all = [ Global; Pure ] in - let all_with_strings = List.map all ~f:(fun t -> (to_string t, t)) in - let all_strings = List.map all_with_strings ~f:fst in - let doc = - sprintf - "How to initialize the opam environment when taking the opam \ - repository from a local directory (may only be used along with \ - the --opam-repository-path option). Possible values are %s. \ - '%s' will use the environment associated with the current opam \ - switch. '%s' will use an empty environment. The default is \ - '%s'." - (String.enumerate_and all_strings) - (to_string Global) (to_string Pure) (to_string default) - in - Arg.( - value - & opt (some (enum all_with_strings)) None - & info [ "opam-env" ] ~doc) - end + let is_archive name = + List.exists + ~f:(fun suffix -> String.is_suffix ~suffix name) + [ ".tar.gz"; ".tgz"; ".tar.bz2"; ".tbz"; ".zip" ] - let of_source = - let open Dune_pkg.Opam.Env in - function - | Source.Global -> global () - | Pure -> empty - end + let path = + let open Fiber.O in + let ( / ) = Filename.concat in + fun { url } -> + Fiber.of_thunk @@ fun () -> + let target_dir = + Xdg.cache_dir (Lazy.force Dune_util.xdg) / "dune/opam-repository" + in + let target = target_dir |> Path.External.of_string |> Path.external_ in + let unpack = url |> OpamUrl.to_string |> is_archive in + let+ res = Fetch.fetch ~unpack ~checksum:None ~target url in + match res with + | Ok () -> Ok target + | Error _ as failure -> failure +end - let term = - let+ opam_repository_path = - Arg.( - value - & opt (some string) None - & info [ "opam-repository-path" ] ~docv:"PATH" - ~doc: - "Path to a local opam repository. This should be a directory \ - containing a valid opam repository such as the one at \ - https://github.com/ocaml/opam-repository. If this option is \ - omitted the dependencies will be locked using the current \ - switch instead.") - and+ opam_switch_name = +module Lock = struct + module Env = struct + module Source = struct + type t = + | Global + | Pure + + let to_string = function + | Global -> "global" + | Pure -> "pure" + + let default = Global + + let term = + let all = [ Global; Pure ] in + let all_with_strings = List.map all ~f:(fun t -> (to_string t, t)) in + let all_strings = List.map all_with_strings ~f:fst in + let doc = + sprintf + "How to initialize the opam environment when taking the opam \ + repository from a local directory (may only be used along with \ + the --opam-repository-path option). Possible values are %s. '%s' \ + will use the environment associated with the current opam switch. \ + '%s' will use an empty environment. The default is '%s'." + (String.enumerate_and all_strings) + (to_string Global) (to_string Pure) (to_string default) + in Arg.( value - & opt (some string) None - & info [ "opam-switch" ] ~docv:"SWITCH" - ~doc: - "Name or path of opam switch to use while solving \ - dependencies. Local switches may be specified with relative \ - paths (e.g. `--opam-switch=.`)") - and+ env_source = Env.Source.term in - let module Repo_selection = Dune_pkg.Opam.Repo_selection in - match (opam_switch_name, opam_repository_path, env_source) with - | None, None, _env_source | Some _, Some _, _env_source -> - User_error.raise - [ Pp.text - "Exactly one of --opam-switch and --opam-repository-path must be \ - specified" - ] - | Some _opam_switch, None, Some _env_source -> - User_error.raise - [ Pp.text "--opam-env may only used with --opam-repository-path" ] - | Some opam_switch_name, None, None -> - (* switch with name does not support environments *) - Repo_selection.switch_with_name opam_switch_name - | None, Some opam_repo_dir_path, env_source_opt -> - let env = - Option.value env_source_opt ~default:Env.Source.default - |> Env.of_source + & opt (some (enum all_with_strings)) None + & info [ "opam-env" ] ~doc) + end + + let of_source = + let open Dune_pkg.Opam.Env in + function + | Source.Global -> global () + | Pure -> empty + end + + module Opam_repository_path = struct + let term = + let dune_path = + let parser s = + s |> Path.External.of_filename_relative_to_initial_cwd + |> Path.external_ |> Result.ok in - Repo_selection.local_repo_with_env ~opam_repo_dir_path ~env + let printer pf p = Pp.to_fmt pf (Path.pp p) in + Arg.conv (parser, printer) + in + Arg.( + value + & opt (some dune_path) None + & info [ "opam-repository-path" ] ~docv:"PATH" + ~doc: + "Path to a local opam repository. This should be a directory \ + containing a valid opam repository such as the one at \ + https://github.com/ocaml/opam-repository. If this option is \ + omitted the dependencies will be locked using the current \ + switch instead.") + end + + module Opam_repository_url = struct + let term = + let parser s = + match OpamUrl.parse_opt s with + | Some url -> Ok url + | None -> Error (`Msg "URL can't be parsed") + in + let printer pf u = Pp.to_fmt pf (Pp.text (OpamUrl.to_string u)) in + let opam_url = Arg.conv (parser, printer) in + Arg.( + value + & opt (some opam_url) None + & info [ "opam-repository-url" ] ~docv:"URL" + ~doc: + "URL of opam repository to download. Can be either a git \ + repository or a link to the tarball of a repository.") end module Version_preference = struct @@ -244,7 +270,9 @@ module Lock = struct let term = let+ (common : Common.t) = Common.term - and+ repo_selection = Repo_selection.term + and+ env_source = Env.Source.term + and+ opam_repository_path = Opam_repository_path.term + and+ opam_repository_url = Opam_repository_url.term and+ context_name = context_term and+ all_contexts = Arg.( @@ -262,15 +290,33 @@ module Lock = struct in Per_context.check_for_dup_lock_dir_paths per_context; let* source_dir = Memo.run (Source_tree.root ()) in - let project = Source_tree.Dir.project source_dir in - let dune_package_map = Dune_project.packages project in - let opam_file_map = opam_file_map_of_dune_package_map dune_package_map in + let* opam_repo_dir = + match opam_repository_path with + | Some path -> Fiber.return path + | None -> ( + let repo = + opam_repository_url + |> Option.map ~f:Opam_repository.of_url + |> Option.value ~default:Opam_repository.default + in + let+ opam_repository = Opam_repository.path repo in + match opam_repository with + | Ok path -> path + | Error _ -> failwith "TODO") + in + let env = + Option.value env_source ~default:Env.Source.default |> Env.of_source + in let+ sys_env = Dune_pkg.Sys_poll.sys_env ~path:(Env_path.path Stdune.Env.initial) in + let env = Dune_pkg.Opam.Env.union env sys_env in let repo_selection = - Dune_pkg.Opam.Repo_selection.add_env sys_env repo_selection + Repo_selection.local_repo_with_env ~opam_repo_dir ~env in + let project = Source_tree.Dir.project source_dir in + let dune_package_map = Dune_project.packages project in + let opam_file_map = opam_file_map_of_dune_package_map dune_package_map in (* Construct a list of thunks that will perform all the file IO side effects after performing validation so that if materializing any lockdir would fail then no side effect takes place. *) diff --git a/src/dune_pkg/fetch.ml b/src/dune_pkg/fetch.ml index 1aeb25a80ab..317a6920044 100644 --- a/src/dune_pkg/fetch.ml +++ b/src/dune_pkg/fetch.ml @@ -67,15 +67,15 @@ type failure = | Checksum_mismatch of Checksum.t | Unavailable of User_message.t option -let fetch ~checksum ~target (url : OpamUrl.t) = +let fetch ~unpack ~checksum ~target (url : OpamUrl.t) = let open Fiber.O in let path = Path.to_string target in let pull = (* hashes have to be empty otherwise OPAM deletes the file after downloading if the hash does not match *) let hashes = [] in - match url.backend with - | #OpamUrl.version_control -> ( + match (url.backend, unpack) with + | #OpamUrl.version_control, _ | _, true -> ( let dirname = OpamFilename.Dir.of_string path in fun label url -> let open OpamProcess.Job.Op in diff --git a/src/dune_pkg/fetch.mli b/src/dune_pkg/fetch.mli index 0184dd50ac9..aab43acb28b 100644 --- a/src/dune_pkg/fetch.mli +++ b/src/dune_pkg/fetch.mli @@ -13,7 +13,8 @@ type failure = @raise Unavailable When the file can't be retrieved, e.g. not available at the location. *) val fetch : - checksum:Checksum.t option + unpack:bool + -> checksum:Checksum.t option -> target:Path.t -> OpamUrl.t -> (unit, failure) result Fiber.t diff --git a/src/dune_pkg/opam.ml b/src/dune_pkg/opam.ml index 50701485c97..daaa60d9075 100644 --- a/src/dune_pkg/opam.ml +++ b/src/dune_pkg/opam.ml @@ -1,60 +1,6 @@ open Stdune module Package_name = Dune_lang.Package_name -module Global : sig - val with_switch_state : - switch_name:string - -> f:(OpamStateTypes.unlocked OpamStateTypes.switch_state -> 'a) - -> 'a -end = struct - let initialized = ref false - - let ensure_initialized () = - if not !initialized then ( - OpamSystem.init (); - let root = OpamStateConfig.opamroot () in - let (_ : OpamFile.Config.t option) = - OpamStateConfig.load_defaults ~lock_kind:`Lock_read root - in - OpamFormatConfig.init (); - OpamCoreConfig.init ~safe_mode:true (); - OpamRepositoryConfig.init (); - OpamStateConfig.init ~root_dir:root (); - initialized := true) - - let with_switch_state ~switch_name ~f = - ensure_initialized (); - try - OpamGlobalState.with_ `Lock_read (fun global_state -> - Dune_console.print - [ Pp.textf "Using opam installation: %s" - (OpamFilename.Dir.to_string global_state.root) - ]; - let switch = OpamSwitch.of_string switch_name in - if not (OpamGlobalState.switch_exists global_state switch) then - User_error.raise - [ Pp.textf - "There is no opam switch named %s. Run `opam switch list` to \ - see a list of switches." - (String.maybe_quoted switch_name) - ]; - OpamSwitchState.with_ ~switch `Lock_read global_state - (fun switch_state -> - Dune_console.print - [ Pp.textf "Using opam switch: %s" - (OpamSwitch.to_string switch_state.switch) - ]; - f switch_state)) - with - | OpamPp.Bad_version _ as bad_version -> - User_error.raise [ Pp.text (OpamPp.string_of_bad_format bad_version) ] - | OpamGlobalState.Configuration_error configuration_error -> - User_error.raise - [ Pp.text - (OpamGlobalState.string_of_configuration_error configuration_error) - ] -end - module Local_repo = struct let ( / ) = Filename.concat @@ -187,21 +133,15 @@ end module Solver = Opam_0install.Solver.Make (Opam_solver.Context_for_dune) module Repo_state = struct - type t = - | Switch of OpamStateTypes.unlocked OpamStateTypes.switch_state - | Local_repo_with_env of Local_repo_with_env.t + type t = Local_repo_with_env of Local_repo_with_env.t let load_opam_package t opam_package = match t with - | Switch switch_state -> OpamSwitchState.opam switch_state opam_package | Local_repo_with_env { local_repo; _ } -> Local_repo.load_opam_package local_repo opam_package let create_context t local_packages ~solver_env ~version_preference = match t with - | Switch switch_state -> - Opam_solver.Context_for_dune.create_switch_context ~solver_env - ~local_packages ~switch_state ~version_preference | Local_repo_with_env { local_repo = { packages_dir_path }; env } -> Opam_solver.Context_for_dune.create_dir_context ~solver_env ~env:(fun name -> Env.find_by_name env ~name) @@ -209,37 +149,20 @@ module Repo_state = struct end module Repo_selection = struct - type pre = - | Switch_with_name of string - | Local_repo_with_env of Local_repo_with_env.t - - type t = pre + type t = Local_repo_with_env.t - let switch_with_name switch_name = Switch_with_name switch_name - - let local_repo_with_env ~opam_repo_dir_path ~env = - Local_repo_with_env - { Local_repo_with_env.local_repo = - Local_repo.of_opam_repo_dir_path opam_repo_dir_path - ; env - } - - let add_env env' = function - | Switch_with_name _ as rs -> rs - | Local_repo_with_env ({ env; _ } as local) -> - let env = Env.union env env' in - let local = { local with env } in - Local_repo_with_env local + let local_repo_with_env ~opam_repo_dir ~env = + let opam_repo_dir_path = Path.to_string opam_repo_dir in + { Local_repo_with_env.local_repo = + Local_repo.of_opam_repo_dir_path opam_repo_dir_path + ; env + } (* [with_state ~f t] calls [f] on a [Repo_state.t] implied by [t] and returns the result. Don't let the [Repo_state.t] escape the function [f]. *) - let with_state ~f = function - | Switch_with_name switch_name -> - Global.with_switch_state ~switch_name ~f:(fun switch_state -> - f (Repo_state.Switch switch_state)) - | Local_repo_with_env local_repo_with_env -> - f (Repo_state.Local_repo_with_env local_repo_with_env) + let with_state ~f local_repo_with_env = + f (Repo_state.Local_repo_with_env local_repo_with_env) end module Summary = struct diff --git a/src/dune_pkg/opam.mli b/src/dune_pkg/opam.mli index 164b11369a4..41de247ad11 100644 --- a/src/dune_pkg/opam.mli +++ b/src/dune_pkg/opam.mli @@ -23,36 +23,8 @@ module Repo_selection : sig (** An opam repository *) type t - (** A t that does not yet have an environment added. Use [add_env]. *) - type pre - - (** An opam repo associated with a switch of a given name or directory *) - val switch_with_name : string -> pre - - (** An opam repo in a local directory given by [opam_repo_dir_path]. Note that - [opam_repo_dir_path] is a [Filename.t] rather than, say, a - [Path.Outside_build_dir.t]. This is due to a current limitation in what - that type (and other modules in [Path]) can represent. It's desirable that - the [opam_repo_dir_path] argument have the following properties: - - - It should be possible to build a [Repo_selection.t] from command line - arguments during argument parsing, which precludes the use of a [Path.t] - here as creating a [Path.t] requires some global state to be set up (by - [Common.init]) which isn't setup at the time command line arguments are - parsed. - - This argument should accept both paths within the source directory and - external paths, since in practical usage it will usually be pointed to - some out-of-source copy of the opam repository, but for unit tests it's - convenient that this support paths within the source directory too. This - rules out using [Path.External.t], [Path.Source.t], and [Path.Local.t]. - - We want to support relative paths external to the source directory to - handle the case where someone has the opam repository checkoud out next - to their project, and runs: `dune pkg lock --opam-repository-path - ../opam-repository`. This rules out [Path.Outside_build_dir.t] as it - doesn't permit relative paths outside the source directory. *) - val local_repo_with_env : opam_repo_dir_path:Filename.t -> env:Env.t -> pre - - val add_env : Env.t -> pre -> t + (** An opam repo in a local directory given by [opam_repo_dir]. *) + val local_repo_with_env : opam_repo_dir:Path.t -> env:Env.t -> t end module Summary : sig diff --git a/src/dune_pkg/opam_solver.ml b/src/dune_pkg/opam_solver.ml index 500e3efc37a..447cf968ebf 100644 --- a/src/dune_pkg/opam_solver.ml +++ b/src/dune_pkg/opam_solver.ml @@ -2,33 +2,6 @@ open Stdune module type CONTEXT = Opam_0install.S.CONTEXT -(* Helper functor which implements [CONTEXT] for [(L.t, R.t) Either.t] where - [L] and [R] both implement [CONTEXT] *) -module Context_either (L : CONTEXT) (R : CONTEXT) : - CONTEXT with type t = (L.t, R.t) Either.t = struct - type t = (L.t, R.t) Either.t - - type rejection = (L.rejection, R.rejection) Either.t - - let pp_rejection f = Either.map ~l:(L.pp_rejection f) ~r:(R.pp_rejection f) - - let candidates = - let convert_rejections ~f = - List.map ~f:(fun (version, result) -> - (version, Result.map_error ~f result)) - in - Either.map - ~l:(fun l name -> - L.candidates l name |> convert_rejections ~f:Either.left) - ~r:(fun r name -> - R.candidates r name |> convert_rejections ~f:Either.right) - - let user_restrictions = - Either.map ~l:L.user_restrictions ~r:R.user_restrictions - - let filter_deps = Either.map ~l:L.filter_deps ~r:R.filter_deps -end - (* Helper module for working with [OpamTypes.filtered_formula] *) module Filtered_formula : sig open OpamTypes @@ -114,9 +87,7 @@ end module Context_for_dune = struct module Dir_context = Opam_0install.Dir_context - module Switch_context = Opam_0install.Switch_context - include - Context_with_local_packages (Context_either (Dir_context) (Switch_context)) + include Context_with_local_packages (Dir_context) let prefer_oldest = function | Version_preference.Oldest -> true @@ -129,14 +100,5 @@ module Context_for_dune = struct ~prefer_oldest:(prefer_oldest version_preference) ~constraints:OpamPackage.Name.Map.empty ~env packages_dir_path in - create ~base_context:(Left dir_context) ~local_packages ~solver_env - - let create_switch_context ~solver_env ~switch_state ~local_packages - ~version_preference = - let switch_context = - Switch_context.create - ~prefer_oldest:(prefer_oldest version_preference) - ~constraints:OpamPackage.Name.Map.empty switch_state - in - create ~base_context:(Right switch_context) ~local_packages ~solver_env + create ~base_context:dir_context ~local_packages ~solver_env end diff --git a/src/dune_pkg/opam_solver.mli b/src/dune_pkg/opam_solver.mli index db59a6aca62..9136ba43fa6 100644 --- a/src/dune_pkg/opam_solver.mli +++ b/src/dune_pkg/opam_solver.mli @@ -10,11 +10,4 @@ module Context_for_dune : sig -> local_packages:OpamFile.OPAM.t OpamTypes.name_map -> version_preference:Version_preference.t -> t - - val create_switch_context : - solver_env:Solver_env.t - -> switch_state:OpamStateTypes.unlocked OpamStateTypes.switch_state - -> local_packages:OpamFile.OPAM.t OpamTypes.name_map - -> version_preference:Version_preference.t - -> t end diff --git a/src/dune_rules/pkg_rules.ml b/src/dune_rules/pkg_rules.ml index 393017153e1..1d299a60001 100644 --- a/src/dune_rules/pkg_rules.ml +++ b/src/dune_rules/pkg_rules.ml @@ -1043,8 +1043,8 @@ module Fetch = struct let* () = Fiber.return () in let* res = let checksum = Option.map checksum ~f:snd in - Dune_pkg.Fetch.fetch ~checksum ~target:(Path.build target_dir) - (OpamUrl.of_string url) + Dune_pkg.Fetch.fetch ~unpack:false ~checksum + ~target:(Path.build target_dir) (OpamUrl.of_string url) in match res with | Ok () -> Fiber.return () diff --git a/test/blackbox-tests/test-cases/pkg/detect-duplicate-lock-dir-paths.t b/test/blackbox-tests/test-cases/pkg/detect-duplicate-lock-dir-paths.t index dc1245cf0b7..2c60135e6af 100644 --- a/test/blackbox-tests/test-cases/pkg/detect-duplicate-lock-dir-paths.t +++ b/test/blackbox-tests/test-cases/pkg/detect-duplicate-lock-dir-paths.t @@ -15,21 +15,21 @@ Define several build contexts that all use the default lockdir > EOF Check that we can still generate lockdirs for individual contexts: - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Solution for dune.lock: (no dependencies to lock) - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=default + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=default Solution for dune.lock: (no dependencies to lock) - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=custom-context-with-default-lock-dir + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=custom-context-with-default-lock-dir Solution for dune.lock: (no dependencies to lock) It's an error to use --all-contexts when there are multiple contexts with the same lockdir: - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --all-contexts + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --all-contexts File "dune-workspace", line 5, characters 1-56: 5 | (default 6 | (name custom-context-with-default-lock-dir))) @@ -54,17 +54,17 @@ Define several build contexts that all use the same custom lockdir: > EOF Check that we can still generate lockdirs for individual contexts: - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=a + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=a Solution for foo.lock: (no dependencies to lock) - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=b + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=b Solution for foo.lock: (no dependencies to lock) It's an error to use --all-contexts when there are multiple contexts with the same lockdir: - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --all-contexts + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --all-contexts File "dune-workspace", line 7, characters 1-39: 7 | (default 8 | (name a) diff --git a/test/blackbox-tests/test-cases/pkg/invalid-opam-repo-errors.t/run.t b/test/blackbox-tests/test-cases/pkg/invalid-opam-repo-errors.t/run.t index 58674ff2992..0b7f5ac0463 100644 --- a/test/blackbox-tests/test-cases/pkg/invalid-opam-repo-errors.t/run.t +++ b/test/blackbox-tests/test-cases/pkg/invalid-opam-repo-errors.t/run.t @@ -6,36 +6,49 @@ Test the error cases for invalid opam repositories > (name lockfile_generation_test)) > EOF - $ dune pkg lock --opam-env=pure --opam-repository=directory-that-does-not-exist - Error: directory-that-does-not-exist does not exist + $ dune pkg lock --opam-env=pure --opam-repository-path=directory-that-does-not-exist + Error: + $TESTCASE_ROOT/directory-that-does-not-exist + does not exist [1] $ touch empty - $ dune pkg lock --opam-env=pure --opam-repository=empty - Error: empty is not a directory + $ dune pkg lock --opam-env=pure --opam-repository-path=empty + Error: + $TESTCASE_ROOT/empty + is not a directory [1] - $ dune pkg lock --opam-env=pure --opam-repository=no-packages-dir - Error: no-packages-dir doesn't look like a path to an opam repository as it - lacks a subdirectory named "packages" + $ dune pkg lock --opam-env=pure --opam-repository-path=no-packages-dir + Error: + $TESTCASE_ROOT/no-packages-dir + doesn't look like a path to an opam repository as it lacks a subdirectory + named "packages" [1] - $ dune pkg lock --opam-env=pure --opam-repository=no-repo-file - Error: File no-repo-file/repo does not exist or can't be read + $ dune pkg lock --opam-env=pure --opam-repository-path=no-repo-file + Error: File + $TESTCASE_ROOT/no-repo-file/repo + does not exist or can't be read [1] - $ dune pkg lock --opam-env=pure --opam-repository=bad-repo-file - Error: At bad-repo-file/repo:1:4-1:8:: + $ dune pkg lock --opam-env=pure --opam-repository-path=bad-repo-file + Error: At + $TESTCASE_ROOT/bad-repo-file/repo:1:4-1:8:: Parse error [1] - $ dune pkg lock --opam-env=pure --opam-repository=no-repo-version - Error: The file no-repo-version/repo lacks an "opam-version" field. + $ dune pkg lock --opam-env=pure --opam-repository-path=no-repo-version + Error: The file + $TESTCASE_ROOT/no-repo-version/repo + lacks an "opam-version" field. Hint: Add `opam-version: "2.0"` to the file. [1] - $ dune pkg lock --opam-env=pure --opam-repository=bad-repo-version - Error: The file bad-repo-version/repo specifies an opam-version which is too - low (1.0). The minimum opam-version is 2.0. + $ dune pkg lock --opam-env=pure --opam-repository-path=bad-repo-version + Error: The file + $TESTCASE_ROOT/bad-repo-version/repo + specifies an opam-version which is too low (1.0). The minimum opam-version is + 2.0. Hint: Change the opam-version field to `opam-version: "2.0"`. [1] diff --git a/test/blackbox-tests/test-cases/pkg/lock-directory-regeneration-safety.t/run.t b/test/blackbox-tests/test-cases/pkg/lock-directory-regeneration-safety.t/run.t index 7fcb328b979..1a488df6913 100644 --- a/test/blackbox-tests/test-cases/pkg/lock-directory-regeneration-safety.t/run.t +++ b/test/blackbox-tests/test-cases/pkg/lock-directory-regeneration-safety.t/run.t @@ -1,5 +1,5 @@ Create a lock directory that didn't originally exist - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Solution for dune.lock: (no dependencies to lock) @@ -7,7 +7,7 @@ Create a lock directory that didn't originally exist (lang package 0.1) Re-create a lock directory in the newly created lock dir - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Solution for dune.lock: (no dependencies to lock) @@ -17,7 +17,7 @@ Re-create a lock directory in the newly created lock dir Attempt to create a lock directory inside an existing directory without a lock.dune file $ rm -rf dune.lock $ cp -r dir-without-metadata dune.lock - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Error: Refusing to regenerate lock directory dune.lock Specified lock dir lacks metadata file (lock.dune) [1] @@ -25,7 +25,7 @@ Attempt to create a lock directory inside an existing directory without a lock.d Attempt to create a lock directory inside an existing directory with an invalid lock.dune file $ rm -rf dune.lock $ cp -r dir-with-invalid-metadata dune.lock - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Error: Refusing to regenerate lock directory dune.lock File "dune.lock/lock.dune", line 1, characters 0-12: Error: Invalid first line, expected: (lang ) @@ -35,7 +35,7 @@ Attempt to create a lock directory inside an existing directory with an invalid Attempt to create a lock directory with the same name as an existing regular file $ rm -rf dune.lock $ touch dune.lock - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Error: Refusing to regenerate lock directory dune.lock Specified lock dir path is not a directory [1] diff --git a/test/blackbox-tests/test-cases/pkg/lock-per-context.t/run.t b/test/blackbox-tests/test-cases/pkg/lock-per-context.t/run.t index 406f3736409..7351b261fa4 100644 --- a/test/blackbox-tests/test-cases/pkg/lock-per-context.t/run.t +++ b/test/blackbox-tests/test-cases/pkg/lock-per-context.t/run.t @@ -32,22 +32,22 @@ Generate a `dune-project` file listing some dependencies. > EOF Test that we get an error when --context and --all-contexts are passed at the same time. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --all-contexts --context=foo + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --all-contexts --context=foo Error: --context and --all-contexts are mutually exclusive [1] Test that we get an error if a non-existant context is specified. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=baz + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=baz Error: Unknown build context: baz [1] Test that we get an error if an opam context is specified. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=bar + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=bar Error: Unexpected opam build context: bar [1] Generate the lockdir for the default context. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Solution for foo.lock: bar.0.5.0 baz.0.1.0 @@ -64,7 +64,7 @@ Only foo.lock (the default context's lockdir) was generated. $ rm -rf *.lock Generate the lockdir with the default context explicitly specified. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=default + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=default Solution for foo.lock: bar.0.5.0 baz.0.1.0 @@ -81,7 +81,7 @@ Again, only foo.lock (the default context's lockdir) was generated. $ rm -rf *.lock Generate the lockdir for the non-default context. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=foo + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=foo Solution for bar.lock: bar.0.5.0 baz.0.1.0 @@ -98,7 +98,7 @@ Now only bar.lock was generated. $ rm -rf *.lock Generate the lockdir for a context which prefers oldest package versions. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=prefers_oldest + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=prefers_oldest Solution for prefers_oldest.lock: bar.0.4.0 baz.0.1.0 @@ -107,7 +107,7 @@ Generate the lockdir for a context which prefers oldest package versions. Re-generate the lockdir for a context which prefers oldest package versions, but override it to prefer newest with a command line argument. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --context=prefers_oldest --version-preference=newest + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --context=prefers_oldest --version-preference=newest Solution for prefers_oldest.lock: bar.0.5.0 baz.0.1.0 @@ -115,7 +115,7 @@ but override it to prefer newest with a command line argument. Generate the lockdir for all (non-opam) contexts. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --all-contexts + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --all-contexts Solution for prefers_oldest.lock: bar.0.4.0 baz.0.1.0 diff --git a/test/blackbox-tests/test-cases/pkg/lockfile-generation.t/run.t b/test/blackbox-tests/test-cases/pkg/lockfile-generation.t/run.t index 1109bc0e21d..6d2842fc005 100644 --- a/test/blackbox-tests/test-cases/pkg/lockfile-generation.t/run.t +++ b/test/blackbox-tests/test-cases/pkg/lockfile-generation.t/run.t @@ -12,7 +12,7 @@ Generate a `dune-project` file. > EOF Run the solver and generate a lock directory. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Solution for dune.lock: bar.0.5.0 baz.0.1.0 @@ -56,7 +56,7 @@ Print the contents of each file in the lockdir: Run the solver again preferring oldest versions of dependencies: - $ dune pkg lock --opam-env=pure --version-preference=oldest --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --version-preference=oldest --opam-repository-path=mock-opam-repository Solution for dune.lock: bar.0.4.0 baz.0.1.0 @@ -106,7 +106,7 @@ Regenerate the `dune-project` file introducing an unsatisfiable constraint. > EOF Run the solver again. This time it will fail. - $ dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + $ dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository Error: Can't find all required versions. Selected: baz.0.1.0 foo.0.0.1 lockfile_generation_test.dev - bar -> (problem) diff --git a/test/blackbox-tests/test-cases/pkg/opam-repository-download.t b/test/blackbox-tests/test-cases/pkg/opam-repository-download.t new file mode 100644 index 00000000000..64887f1cc6c --- /dev/null +++ b/test/blackbox-tests/test-cases/pkg/opam-repository-download.t @@ -0,0 +1,101 @@ +Helper shell function that generates an opam file for a package: + + $ mkpkg() { + > name=$1 + > mkdir -p mock-opam-repository/packages/$name/$name.0.0.1 + > cat >mock-opam-repository/packages/$name/$name.0.0.1/opam + > } + +Make a mock repo tarball that will get used by dune to download the package + + $ mkdir mock-opam-repository + $ cat > mock-opam-repository/repo < opam-version: "2.0" + > EOF + $ mkpkg foo < opam-version: "2.0" + > EOF + $ mkpkg bar < opam-version: "2.0" + > depends: [ "foo" ] + > EOF + $ cd mock-opam-repository + $ tar czf ../index.tar.gz * + $ cd .. + +Create a dune project with a minimal single-request HTTP server. This is built and used to +serve our opam-repository as a mock. + + $ cat > dune-project < (lang dune 3.8) + > (generate_opam_files true) + > + > (package + > (name baz) + > (depends + > bar)) + > EOF + $ cat > dune < (executable + > (public_name server) + > (libraries stdune) + > ;; stdune is showing the unstable alert but since this is part of the + > ;; dune codebase, this test has to be adjusted in case the API changes + > (flags -alert -unstable)) + > EOF + > cat > server.ml < open Stdune + > let serve_once ~filename ~port () = + > let host = Unix.inet_addr_loopback in + > let addr = Unix.ADDR_INET (host, port) in + > let sock = Unix.socket ~cloexec:true Unix.PF_INET Unix.SOCK_STREAM 0 in + > Unix.setsockopt sock Unix.SO_REUSEADDR true; + > Unix.bind sock addr; + > Unix.listen sock 5; + > (* signal we're done setting up *) + > let pid = open_out "server.pid" in + > close_out pid; + > let descr, _sockaddr = Unix.accept sock in + > let content = Io.String_path.read_file filename in + > let content_length = String.length content in + > let write_end = Unix.out_channel_of_descr descr in + > Printf.fprintf write_end {|HTTP/1.1 200 Ok + > Content-Length: %d + > + > %s|} + > content_length content; + > close_out write_end + > + > let () = + > let port = int_of_string (Sys.argv.(1)) in + > serve_once ~filename:"index.tar.gz" ~port () + > EOF + $ PORT=$(shuf -i 2048-65000 -n 1) + $ dune exec -- ./server.exe $PORT & + $ # wait for server to come up, this takes a bit + $ while [ ! -f server.pid ]; do sleep 0.01; done + +Run Dune and tell it to cache into our custom cache folder. + + $ mkdir fake-xdg-cache + $ XDG_CACHE_HOME=$(pwd)/fake-xdg-cache dune pkg lock --opam-repository-url=http://localhost:$PORT/index.tar.gz + Solution for dune.lock: + bar.0.0.1 + foo.0.0.1 + + +Our custom cache folder should be populated with the unpacked tarball +containing the repository: + + $ find fake-xdg-cache | sort + fake-xdg-cache + fake-xdg-cache/dune + fake-xdg-cache/dune/opam-repository + fake-xdg-cache/dune/opam-repository/packages + fake-xdg-cache/dune/opam-repository/packages/bar + fake-xdg-cache/dune/opam-repository/packages/bar/bar.0.0.1 + fake-xdg-cache/dune/opam-repository/packages/bar/bar.0.0.1/opam + fake-xdg-cache/dune/opam-repository/packages/foo + fake-xdg-cache/dune/opam-repository/packages/foo/foo.0.0.1 + fake-xdg-cache/dune/opam-repository/packages/foo/foo.0.0.1/opam + fake-xdg-cache/dune/opam-repository/repo diff --git a/test/blackbox-tests/test-cases/pkg/solver-flags-per-context.t b/test/blackbox-tests/test-cases/pkg/solver-flags-per-context.t index 1ac2a6a6c72..22f49e1ae67 100644 --- a/test/blackbox-tests/test-cases/pkg/solver-flags-per-context.t +++ b/test/blackbox-tests/test-cases/pkg/solver-flags-per-context.t @@ -9,7 +9,7 @@ Helper shell function that generates an opam file for a package: Helper shell function to generate a dune-project file and generate lockdir: $ solve_project() { > cat >dune-project - > dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository --all-contexts + > dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository --all-contexts > } Create a workspace file with some contexts with different combinations of with-test and with-doc flags diff --git a/test/blackbox-tests/test-cases/pkg/with-test-dependencies.t b/test/blackbox-tests/test-cases/pkg/with-test-dependencies.t index 1fedf85b57c..1795c054875 100644 --- a/test/blackbox-tests/test-cases/pkg/with-test-dependencies.t +++ b/test/blackbox-tests/test-cases/pkg/with-test-dependencies.t @@ -48,7 +48,7 @@ Generate a mock opam repository including some test dependencies: Helper shell function to generate a dune-project file and generate lockdir: $ solve_project() { > cat >dune-project - > dune pkg lock --opam-env=pure --opam-repository=mock-opam-repository + > dune pkg lock --opam-env=pure --opam-repository-path=mock-opam-repository > } Regular dependencies are resolved transitively: diff --git a/test/expect-tests/dune_pkg/dune_pkg_unit_tests.ml b/test/expect-tests/dune_pkg/dune_pkg_unit_tests.ml index 0660ecd4d5f..24264c659e4 100644 --- a/test/expect-tests/dune_pkg/dune_pkg_unit_tests.ml +++ b/test/expect-tests/dune_pkg/dune_pkg_unit_tests.ml @@ -39,7 +39,7 @@ let wrong_checksum = let download ~port ~filename ~target ?checksum () = let open Fiber.O in let url = url ~port ~filename in - let* res = Fetch.fetch ~checksum ~target url in + let* res = Fetch.fetch ~unpack:false ~checksum ~target url in match res with | Error (Unavailable None) -> let errs = [ Pp.text "Failure while downloading" ] in