Add track_content option to allow hashing of include_dependencys

base/loading.jl

@@ -1780,31 +1780,34 @@ function _include_dependency(mod::Module, _path::AbstractString; track_content=t
     if _track_dependencies[]
         @lock require_lock begin
             if track_content
-                @assert isfile(path) "can only hash files"
+                hash = isdir(path) ? _crc32c(join(readdir(path))) : open(_crc32c, path, "r")
                 # use mtime=-1.0 here so that fsize==0 && mtime==0.0 corresponds to a missing include_dependency
-                push!(_require_dependencies,
-                      (mod, path, filesize(path), open(_crc32c, path, "r"), -1.0))
+                push!(_require_dependencies, (mod, path, filesize(path), hash, -1.0))
             else
-                push!(_require_dependencies,
-                      (mod, path, UInt64(0), UInt32(0), mtime(path)))
+                push!(_require_dependencies, (mod, path, UInt64(0), UInt32(0), mtime(path)))
             end
         end
     end
     return path, prev
 end
 
 """
-    include_dependency(path::AbstractString)
+    include_dependency(path::AbstractString; track_content::Bool=false)
 
 In a module, declare that the file, directory, or symbolic link specified by `path`
 (relative or absolute) is a dependency for precompilation; that is, the module will need
-to be recompiled if the modification time of `path` changes.
+to be recompiled if the modification time `mtime` of `path` changes.
+If `track_content=true` recompilation is triggered when the content of `path` changes
+(if `path` is a directory the content equals `join(readdir(path))`).
 
 This is only needed if your module depends on a path that is not used via [`include`](@ref). It has
 no effect outside of compilation.
+
+!!! compat "Julia 1.11"
+    Keyword argument `track_content` requires at least Julia 1.11.
 """
-function include_dependency(path::AbstractString)
-    _include_dependency(Main, path, track_content=false)
+function include_dependency(path::AbstractString; track_content::Bool=false)
+    _include_dependency(Main, path, track_content=track_content)
     return nothing
 end
 
@@ -2754,7 +2757,7 @@ end
 function resolve_depot(inc::AbstractString)
     startswith(inc, string("@depot", Filesystem.pathsep())) || return :not_relocatable
     for depot in DEPOT_PATH
-        isfile(restore_depot_path(inc, depot)) && return depot
+        ispath(restore_depot_path(inc, depot)) && return depot
     end
     return :no_depot_found
 end
@@ -3480,7 +3483,7 @@ end
                         record_reason(reasons, "include_dependency fsize change")
                         return true
                     end
-                    hash = open(_crc32c, f, "r")
+                    hash = isdir(f) ? _crc32c(join(readdir(f))) : open(_crc32c, f, "r")
                     if hash != hash_req
                         @debug "Rejecting stale cache file $cachefile because hash of $f has changed (hash $hash, before $hash_req)"
                         record_reason(reasons, "include_dependency fhash change")

test/Makefile

@@ -37,24 +37,26 @@ $(addprefix revise-, $(TESTS)): revise-% :
 relocatedepot:
 	@rm -rf $(SRCDIR)/relocatedepot
 	@cd $(SRCDIR) && \
-	$(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
+	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
 	@mkdir $(SRCDIR)/relocatedepot
 	@cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot
 	@cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot
 	@cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot
+	@cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot
 	@cd $(SRCDIR) && \
-	$(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
+	$(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
 
 revise-relocatedepot: revise-% :
 	@rm -rf $(SRCDIR)/relocatedepot
 	@cd $(SRCDIR) && \
-	$(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*)
+	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*)
 	@mkdir $(SRCDIR)/relocatedepot
 	@cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot
 	@cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot
 	@cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot
+	@cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot
 	@cd $(SRCDIR) && \
-	$(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*)
+	$(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*)
 
 embedding:
 	@$(MAKE) -C $(SRCDIR)/$@ check $(EMBEDDING_ARGS)

test/RelocationTestPkg1/Project.toml

@@ -1,4 +1,3 @@
 name = "RelocationTestPkg1"
 uuid = "854e1adb-5a97-46bf-a391-1cfe05ac726d"
-authors = ["flo "]
 version = "0.1.0"

test/RelocationTestPkg2/Project.toml

@@ -1,4 +1,3 @@
 name = "RelocationTestPkg2"
 uuid = "8d933983-b090-4b0b-a37e-c34793f459d1"
-authors = ["flo "]
 version = "0.1.0"

test/RelocationTestPkg2/src/RelocationTestPkg2.jl

@@ -1,6 +1,7 @@
 module RelocationTestPkg2
 
 include_dependency("foo.txt")
+include_dependency("foodir")
 greet() = print("Hello World!")
 
 end # module RelocationTestPkg2

test/RelocationTestPkg3/Project.toml

@@ -0,0 +1,3 @@
+name = "RelocationTestPkg3"
+uuid = "1ba4f954-9da9-4cd2-9ca7-6250235df52c"
+version = "0.1.0"

test/RelocationTestPkg3/src/RelocationTestPkg3.jl

@@ -0,0 +1,7 @@
+module RelocationTestPkg3
+
+include_dependency("bar.txt", track_content=true)
+include_dependency("bardir", track_content=true)
+greet() = print("Hello World!")
+
+end # module RelocationTestPkg3

test/loading.jl

@@ -1504,3 +1504,31 @@ end
         @test occursin(r"Loading object cache file .+ for Parent", log)
     end
 end
+
+@testset "including non-existent file throws proper error #52462" begin
+    mktempdir() do depot
+        project_path = joinpath(depot, "project")
+        mkpath(project_path)
+
+        # Create a `Foo.jl` package
+        foo_path = joinpath(depot, "dev", "Foo52462")
+        mkpath(joinpath(foo_path, "src"))
+        open(joinpath(foo_path, "src", "Foo52462.jl"); write=true) do io
+            println(io, """
+            module Foo52462
+            include("non-existent.jl")
+            end
+            """)
+        end
+        open(joinpath(foo_path, "Project.toml"); write=true) do io
+            println(io, """
+            name = "Foo52462"
+            uuid = "00000000-0000-0000-0000-000000000001"
+            version = "1.0.0"
+            """)
+        end
+
+        file = joinpath(depot, "dev", "non-existent.jl")
+        @test_throws SystemError("opening file $(repr(file))") include(file)
+    end
+end

test/relocatedepot.jl

@@ -1,5 +1,4 @@
 using Test
-using Logging
 
 
 include("testenv.jl")
@@ -18,6 +17,10 @@ function test_harness(@nospecialize(fn); empty_load_path=true, empty_depot_path=
     end
 end
 
+# We test relocation with three dummy pkgs:
+# - RelocationTestPkg1 - no include_dependency
+# - RelocationTestPkg2 - with include_dependency tracked by `mtime`
+# - RelocationTestPkg3 - with include_dependency tracked by content
 
 if !test_relocated_depot
 
@@ -70,27 +73,46 @@ if !test_relocated_depot
         pkgname = "RelocationTestPkg1"
         test_harness(empty_depot_path=false) do
             push!(LOAD_PATH, @__DIR__)
-            push!(DEPOT_PATH, @__DIR__) # required to make relocatable, but cache is written to DEPOT_PATH[1]
+            push!(DEPOT_PATH, @__DIR__) # make src files available for relocation
             pkg = Base.identify_package(pkgname)
             cachefiles = Base.find_all_in_cache_path(pkg)
             rm.(cachefiles, force=true)
             @test Base.isprecompiled(pkg) == false
-            Base.require(pkg) # precompile
+            Base.require(pkg)
             @test Base.isprecompiled(pkg, ignore_loaded=true) == true
         end
     end
 
-    @testset "precompile RelocationTestPkg2 (contains include_dependency)" begin
+    @testset "precompile RelocationTestPkg2" begin
         pkgname = "RelocationTestPkg2"
         test_harness(empty_depot_path=false) do
             push!(LOAD_PATH, @__DIR__)
-            push!(DEPOT_PATH, @__DIR__) # required to make relocatable, but cache is written to DEPOT_PATH[1]
+            push!(DEPOT_PATH, @__DIR__) # make src files available for relocation
             pkg = Base.identify_package(pkgname)
             cachefiles = Base.find_all_in_cache_path(pkg)
             rm.(cachefiles, force=true)
+            rm(joinpath(@__DIR__, pkgname, "src", "foodir"), force=true, recursive=true)
             @test Base.isprecompiled(pkg) == false
             touch(joinpath(@__DIR__, pkgname, "src", "foo.txt"))
-            Base.require(pkg) # precompile
+            mkdir(joinpath(@__DIR__, pkgname, "src", "foodir"))
+            Base.require(pkg)
+            @test Base.isprecompiled(pkg, ignore_loaded=true) == true
+        end
+    end
+
+    @testset "precompile RelocationTestPkg3" begin
+        pkgname = "RelocationTestPkg3"
+        test_harness(empty_depot_path=false) do
+            push!(LOAD_PATH, @__DIR__)
+            push!(DEPOT_PATH, @__DIR__) # make src files available for relocation
+            pkg = Base.identify_package(pkgname)
+            cachefiles = Base.find_all_in_cache_path(pkg)
+            rm.(cachefiles, force=true)
+            rm(joinpath(@__DIR__, pkgname, "src", "bardir"), force=true, recursive=true)
+            @test Base.isprecompiled(pkg) == false
+            touch(joinpath(@__DIR__, pkgname, "src", "bar.txt"))
+            mkdir(joinpath(@__DIR__, pkgname, "src", "bardir"))
+            Base.require(pkg)
             @test Base.isprecompiled(pkg, ignore_loaded=true) == true
         end
     end
@@ -139,10 +161,8 @@ if !test_relocated_depot
                         version = "1.0.0"
                         """)
                     end
-                    pushfirst!(LOAD_PATH, depot2)
-                    pushfirst!(DEPOT_PATH, depot2)
-                    pkg = Base.identify_package("Example2")
-                    Base.require(pkg)
+                    pushfirst!(LOAD_PATH, depot2); pushfirst!(DEPOT_PATH, depot2)
+                    pkg = Base.identify_package("Example2"); Base.require(pkg)
                     mktempdir() do depot3
                         # precompile Foo in depot3
                         open(joinpath(depot3, "Module52161.jl"), write=true) do io
@@ -157,10 +177,8 @@ if !test_relocated_depot
                             end
                             """)
                         end
-                        pushfirst!(LOAD_PATH, depot3)
-                        pushfirst!(DEPOT_PATH, depot3)
-                        pkg = Base.identify_package("Module52161")
-                        Base.compilecache(pkg)
+                        pushfirst!(LOAD_PATH, depot3); pushfirst!(DEPOT_PATH, depot3)
+                        pkg = Base.identify_package("Module52161"); Base.compilecache(pkg)
                         cachefile = joinpath(depot3, "compiled",
                                              "v$(VERSION.major).$(VERSION.minor)", "Module52161.ji")
                         _, (deps, _, _), _... = Base.parse_cache_header(cachefile)
@@ -195,21 +213,34 @@ else
             push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file
             pkg = Base.identify_package(pkgname)
             @test Base.isprecompiled(pkg) == true
-            Base.require(pkg) # re-precompile
-            @test Base.isprecompiled(pkg) == true
         end
     end
 
-    @testset "load RelocationTestPkg2 (contains include_dependency) from test/relocatedepot" begin
+    @testset "load RelocationTestPkg2 from test/relocatedepot" begin
         pkgname = "RelocationTestPkg2"
         test_harness() do
             push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot"))
             push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files
             push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file
             pkg = Base.identify_package(pkgname)
             @test Base.isprecompiled(pkg) == false # moving depot changes mtime of include_dependency
-            Base.require(pkg) # re-precompile
+            Base.require(pkg)
             @test Base.isprecompiled(pkg) == true
+            touch(joinpath(@__DIR__, "relocatedepot", "RelocationTestPkg2", "src", "foodir", "foofoo"))
+            @test Base.isprecompiled(pkg) == false
+        end
+    end
+
+    @testset "load RelocationTestPkg3 from test/relocatedepot" begin
+        pkgname = "RelocationTestPkg3"
+        test_harness() do
+            push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot"))
+            push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot"))
+            push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file
+            pkg = Base.identify_package(pkgname)
+            @test Base.isprecompiled(pkg) == true
+            touch(joinpath(@__DIR__, "relocatedepot", "RelocationTestPkg3", "src", "bardir", "barbar"))
+            @test Base.isprecompiled(pkg) == false
         end
     end