fix: allow shadowing builtins

doc/changes/8558.md

@@ -0,0 +1,2 @@
+- Allow libraries to shadow OCaml builtin libraries. Previously, builtin
+  libraries would always take precedence. (@rgrinberg, #8558)

src/dune_rules/findlib.ml

@@ -386,19 +386,16 @@ end = struct
   ;;
 
   let lookup_and_load (db : t) name =
-    match Package.Name.Map.find db.builtins name with
-    | Some meta ->
-      (* XXX DUNE4 weird to favor these hardcoded packages over user possibly
-         user defined libraries *)
-      load_builtin db meta >>| Result.ok
+    Monad.List.find_map db.paths ~f:(lookup db name)
+    >>= function
+    | Some m -> Monad.return m
     | None ->
-      Monad.List.find_map db.paths ~f:(lookup db name)
-      >>| (function
-      | Some m -> m
-      | None ->
-        (match Package.Name.to_string name with
-         | "dune" -> Ok builtin_for_dune
-         | _ -> Error Unavailable_reason.Not_found))
+      (match Package.Name.to_string name with
+       | "dune" -> Monad.return (Ok builtin_for_dune)
+       | _ ->
+         (match Package.Name.Map.find db.builtins name with
+          | None -> Monad.return (Error Unavailable_reason.Not_found)
+          | Some meta -> load_builtin db meta >>| Result.ok))
   ;;
 
   let root_packages (db : t) =

test/blackbox-tests/test-cases/describe/describe-workspace-pp.t

@@ -104,7 +104,8 @@ not stable across different setups.
    (executables
     ((names (exe))
      (requires
-      (c9367091ddd9a70d99fc22ede348f17c
+      (c39d8e11db2363236e69af7750ce7b9a
+       c9367091ddd9a70d99fc22ede348f17c
        1f2b5eb300ea716920494385a31bb5fb
        5014e215e204cf8da6c32644cda1b31e
        249b2edaf3cc552a247667041bb5f015
@@ -125,11 +126,19 @@ not stable across different setups.
         (cmt (_build/default/exe/.exe.eobjs/byte/dune__exe__Exe.cmt))
         (cmti ()))))
      (include_dirs (_build/default/exe/.exe.eobjs/byte))))
+   (library
+    ((name compiler-libs)
+     (uid c39d8e11db2363236e69af7750ce7b9a)
+     (local false)
+     (requires ())
+     (source_dir /FINDLIB/ocaml/compiler-libs)
+     (modules ())
+     (include_dirs (/FINDLIB/ocaml/compiler-libs))))
    (library
     ((name compiler-libs.common)
      (uid c9367091ddd9a70d99fc22ede348f17c)
      (local false)
-     (requires ())
+     (requires (c39d8e11db2363236e69af7750ce7b9a))
      (source_dir /FINDLIB/ocaml/compiler-libs)
      (modules ())
      (include_dirs (/FINDLIB/ocaml/compiler-libs))))
@@ -266,11 +275,19 @@ not stable across different setups.
   $ dune describe workspace --lang 0.1 --sanitize-for-tests --with-pps lib
   ((root /WORKSPACE_ROOT)
    (build_context _build/default)
+   (library
+    ((name compiler-libs)
+     (uid c39d8e11db2363236e69af7750ce7b9a)
+     (local false)
+     (requires ())
+     (source_dir /FINDLIB/ocaml/compiler-libs)
+     (modules ())
+     (include_dirs (/FINDLIB/ocaml/compiler-libs))))
    (library
     ((name compiler-libs.common)
      (uid c9367091ddd9a70d99fc22ede348f17c)
      (local false)
-     (requires ())
+     (requires (c39d8e11db2363236e69af7750ce7b9a))
      (source_dir /FINDLIB/ocaml/compiler-libs)
      (modules ())
      (include_dirs (/FINDLIB/ocaml/compiler-libs))))

test/blackbox-tests/test-cases/describe/describe.t

@@ -377,11 +377,19 @@ not stable across different setups.
      (source_dir /FINDLIB/cmdliner)
      (modules ())
      (include_dirs (/FINDLIB/cmdliner))))
+   (library
+    ((name compiler-libs)
+     (uid c39d8e11db2363236e69af7750ce7b9a)
+     (local false)
+     (requires ())
+     (source_dir /FINDLIB/ocaml/compiler-libs)
+     (modules ())
+     (include_dirs (/FINDLIB/ocaml/compiler-libs))))
    (library
     ((name compiler-libs.common)
      (uid c9367091ddd9a70d99fc22ede348f17c)
      (local false)
-     (requires ())
+     (requires (c39d8e11db2363236e69af7750ce7b9a))
      (source_dir /FINDLIB/ocaml/compiler-libs)
      (modules ())
      (include_dirs (/FINDLIB/ocaml/compiler-libs))))
@@ -909,11 +917,19 @@ not stable across different setups.
      (source_dir /FINDLIB/cmdliner)
      (modules ())
      (include_dirs (/FINDLIB/cmdliner))))
+   (library
+    ((name compiler-libs)
+     (uid c39d8e11db2363236e69af7750ce7b9a)
+     (local false)
+     (requires ())
+     (source_dir /FINDLIB/ocaml/compiler-libs)
+     (modules ())
+     (include_dirs (/FINDLIB/ocaml/compiler-libs))))
    (library
     ((name compiler-libs.common)
      (uid c9367091ddd9a70d99fc22ede348f17c)
      (local false)
-     (requires ())
+     (requires (c39d8e11db2363236e69af7750ce7b9a))
      (source_dir /FINDLIB/ocaml/compiler-libs)
      (modules ())
      (include_dirs (/FINDLIB/ocaml/compiler-libs))))

test/blackbox-tests/test-cases/merlin/instrumentation.t/run.t

@@ -40,12 +40,14 @@ up a project with instrumentation and testing checking the merlin config.
   Foo
   ((STDLIB /OCAMLC_WHERE)
    (EXCLUDE_QUERY_DIR)
+   (B lib/bytes)
    (B lib/findlib)
    (B /OCAMLC_WHERE)
    (B
     $TESTCASE_ROOT/_build/default/lib/.foo.objs/byte)
    (B
     $TESTCASE_ROOT/_build/default/ppx/.hello.objs/byte)
+   (S lib/bytes)
    (S lib/findlib)
    (S /OCAMLC_WHERE)
    (S
@@ -58,12 +60,14 @@ up a project with instrumentation and testing checking the merlin config.
   Privmod
   ((STDLIB /OCAMLC_WHERE)
    (EXCLUDE_QUERY_DIR)
+   (B lib/bytes)
    (B lib/findlib)
    (B /OCAMLC_WHERE)
    (B
     $TESTCASE_ROOT/_build/default/lib/.foo.objs/byte)
    (B
     $TESTCASE_ROOT/_build/default/ppx/.hello.objs/byte)
+   (S lib/bytes)
    (S lib/findlib)
    (S /OCAMLC_WHERE)
    (S

test/blackbox-tests/test-cases/merlin/merlin-tests.t/run.t

@@ -9,12 +9,14 @@ CRAM sanitization
   X
   ((STDLIB /OCAMLC_WHERE)
    (EXCLUDE_QUERY_DIR)
+   (B lib/bytes)
    (B lib/findlib)
    (B /OCAMLC_WHERE)
    (B
     $TESTCASE_ROOT/_build/default/exe/.x.eobjs/byte)
    (B
     $TESTCASE_ROOT/_build/default/lib/.foo.objs/public_cmi)
+   (S lib/bytes)
    (S lib/findlib)
    (S /OCAMLC_WHERE)
    (S
@@ -64,10 +66,12 @@ CRAM sanitization
   Foo
   ((STDLIB /OCAMLC_WHERE)
    (EXCLUDE_QUERY_DIR)
+   (B lib/bytes)
    (B lib/findlib)
    (B /OCAMLC_WHERE)
    (B
     $TESTCASE_ROOT/_build/default/lib/.foo.objs/byte)
+   (S lib/bytes)
    (S lib/findlib)
    (S /OCAMLC_WHERE)
    (S
@@ -84,10 +88,12 @@ CRAM sanitization
   Privmod
   ((STDLIB /OCAMLC_WHERE)
    (EXCLUDE_QUERY_DIR)
+   (B lib/bytes)
    (B lib/findlib)
    (B /OCAMLC_WHERE)
    (B
     $TESTCASE_ROOT/_build/default/lib/.foo.objs/byte)
+   (S lib/bytes)
    (S lib/findlib)
    (S /OCAMLC_WHERE)
    (S

test/expect-tests/findlib_tests.ml

@@ -107,9 +107,8 @@ let%expect_test "configurator" =
 
 let%expect_test "builtins" =
   print_pkg_archives "str";
-  [%expect
-    {|
-    Hidden { byte = [ "stdlib/str.cma" ]; native = [ "stdlib/str.cmxa" ] } |}];
+  [%expect {|
+    Available { byte = []; native = [] } |}];
   print_pkg_archives "dynlink";
   [%expect
     {|