fixes

examples/MyApp/Project.toml

@@ -4,8 +4,8 @@ authors = ["Kristoffer Carlsson <kristoffer.carlsson@juliacomputing.com>"]
 version = "0.1.0"
 
 [deps]
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Example = "7876af07-990d-54b4-ab0e-23690620f79a"
 HelloWorldC_jll = "dca1746e-5efc-54fc-8249-22745bc95a49"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

examples/MyApp/src/MyApp.jl

@@ -2,7 +2,7 @@ module MyApp
 
 using Example
 using HelloWorldC_jll
-using Pkg.Artifacts
+using Artifacts
 using Distributed
 
 
@@ -64,10 +64,8 @@ function real_main()
         1
     end
     println("n = $n")
-    # TODO: Code loading for distributed is currently only
-    # really possible by shipping a Project.toml.  
-    # @eval @everywhere using Example
-    # @everywhere println(Example.domath(3))
+    @eval @everywhere using Example
+    @everywhere println(Example.domath(3))
     return
 end
 

src/PackageCompiler.jl

@@ -144,15 +144,12 @@ function get_compat_version_str(version::VersionNumber, level::String)
 end
 
 function get_sysimg_file(name::String;
-                     library_only::Bool=false,
                      version::Union{VersionNumber, Nothing}=nothing,
                      level::String="patch")
 
     dlext = Libdl.dlext
-    (!library_only || Sys.iswindows()) && return "$name.$dlext"
 
     # For libraries on Unix/Apple, make sure the name starts with "lib"
-
     if !startswith(name, "lib")
         name = "lib" * name
     end
@@ -171,12 +168,6 @@ function get_sysimg_file(name::String;
     return sysimg_file
 end
 
-function get_depot_path(root_dir::String, library_only::Bool)
-    # Use <root>/share/julia as the depot path when creating libraries
-    library_only && return joinpath(root_dir, "share", "julia")
-    return root_dir
-end
-
 function get_soname(name::String;
                 library_only::Bool=false,
                 version::Union{VersionNumber, Nothing}=nothing,
@@ -998,12 +989,16 @@ function _create_app(package_dir::String,
     if ctx.env.pkg === nothing
         error("expected package to have a `name`-entry")
     end
-    sysimg_name = ctx.env.pkg.name
+    project_name = ctx.env.pkg.name
     if name === nothing
-        name = sysimg_name
+        name = project_name
     end
 
-    sysimg_file = get_sysimg_file(name, library_only=library_only, version=version)
+    sysimg_file = if library_only
+        get_sysimg_file(name, version=version)
+    else
+        "sys." * Libdl.dlext
+    end
     soname = get_soname(name,
                         library_only=library_only,
                         version=version,
@@ -1023,11 +1018,18 @@ function _create_app(package_dir::String,
 
     bundle_julia_libraries(dest_dir)
     bundle_artifacts(ctx, dest_dir, library_only; include_lazy_artifacts=include_lazy_artifacts)
+    isapp && bundle_julia_executable(dest_dir)
+    # TODO: Should also bundle project and update load_path for library 
+    isapp && bundle_project(ctx, dest_dir)
 
     library_only && bundle_headers(dest_dir, header_files)
 
     # TODO: Create in a temp dir and then move it into place?
-    sysimage_path = Sys.isunix() ? joinpath(dest_dir, "lib") : joinpath(dest_dir, "bin")
+    sysimage_path = if Sys.isunix()
+        isapp ? joinpath(dest_dir, "lib", "julia") : joinpath(dest_dir, "lib")
+    else
+        joinpath(dest_dir, "bin")
+    end
     mkpath(sysimage_path)
     cd(sysimage_path) do
         if !incremental
@@ -1039,7 +1041,7 @@ function _create_app(package_dir::String,
             create_sysimage(String[]; sysimage_path=tmp_base_sysimage, project=package_dir,
                             incremental=false, filter_stdlibs, cpu_target)
 
-            create_sysimage([sysimg_name]; sysimage_path=sysimg_file, project=package_dir,
+            create_sysimage([project_name]; sysimage_path=sysimg_file, project=package_dir,
                             incremental=true,
                             precompile_execution_file,
                             precompile_statements_file,
@@ -1052,7 +1054,7 @@ function _create_app(package_dir::String,
                             sysimage_build_args,
                             include_transitive_dependencies)
         else
-            create_sysimage([sysimg_name]; sysimage_path=sysimg_file, project=package_dir,
+            create_sysimage([project_name]; sysimage_path=sysimg_file, project=package_dir,
                                          incremental, filter_stdlibs,
                                          precompile_execution_file,
                                          precompile_statements_file,
@@ -1072,8 +1074,8 @@ function _create_app(package_dir::String,
         end
 
         if library_only && version !== nothing && Sys.isunix()
-            compat_file = get_sysimg_file(name, library_only=library_only, version=version, level=compat_level)
-            base_file = get_sysimg_file(name, library_only=library_only)
+            compat_file = get_sysimg_file(name, version=version, level=compat_level)
+            base_file = get_sysimg_file(name)
             @debug "creating symlinks for $compat_file and $base_file"
             symlink(sysimg_file, compat_file)
             symlink(sysimg_file, base_file)
@@ -1085,11 +1087,10 @@ function _create_app(package_dir::String,
         mkpath(executable_path)
         cd(executable_path) do
             c_driver_program_path = abspath(c_driver_program)
-            sysimage_path = Sys.iswindows() ? sysimg_file : joinpath("..", "lib", sysimg_file)
+            sysimage_path = Sys.iswindows() ? sysimg_file : joinpath("..", "lib", "julia", sysimg_file)
             create_executable_from_sysimg(; sysimage_path, executable_path=name,
                                          c_driver_program_path)
         end
-        bundle_julia_executable(executable_path)
     end
 
     return
@@ -1107,9 +1108,28 @@ function create_executable_from_sysimg(;sysimage_path::String,
     return nothing
 end
 
+# One of the main reason for bunlding the project file is
+# for Distributed to work. When using Distributed we need to
+# load packages on other workers and that requires the Project file.
+# See https://github.com/JuliaLang/julia/issues/42296 for some discussion.
+function bundle_project(ctx, dir)
+    julia_share =  joinpath(dir, "share", "julia")
+    mkpath(julia_share)
+    # We do not want to bundle some potentially sensitive data, only data that
+    # is already trivially retrievable from the sysimage.
+    d = Dict{String, Any}()
+    d["name"] = ctx.env.project.name
+    d["uuid"] = ctx.env.project.uuid
+    d["deps"] = ctx.env.project.deps
+
+    Pkg.Types.write_project(d, joinpath(julia_share, "Project.toml"))
+end
+
 function bundle_julia_executable(dir::String)
+    bindir = joinpath(dir, "bin")
     name = Sys.iswindows() ? "julia.exe" : "julia"
-    cp(joinpath(Sys.BINDIR::String, name), joinpath(dir, name))
+    mkpath(bindir)
+    cp(joinpath(Sys.BINDIR::String, name), joinpath(bindir, name); force=true)
 end
 
 function bundle_julia_libraries(dest_dir)
@@ -1167,7 +1187,7 @@ function bundle_artifacts(ctx, dest_dir, library_only; include_lazy_artifacts=tr
     end
 
     # Copy the artifacts needed to the app directory
-    depot_path = get_depot_path(dest_dir, library_only)
+    depot_path = joinpath(dest_dir, "share", "julia")
     artifact_app_path = joinpath(depot_path, "artifacts")
 
     if !isempty(artifact_paths)

src/embedding_wrapper.c

@@ -49,19 +49,40 @@ int main(int argc, char *argv[])
     libsupport_init();
 
     // Get the current exe path so we can compute a relative depot path
-    char *free_path = (char*)malloc(PATH_MAX);
+    char *exe_path = (char*)malloc(PATH_MAX);
     size_t path_size = PATH_MAX;
-    if (!free_path)
-       jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno));
-    if (uv_exepath(free_path, &path_size)) {
-       jl_error("fatal error: unexpected error while retrieving exepath");
+    if (!exe_path) {
+        jl_errorf("fatal error: failed to allocate memory: %s", strerror(errno));
+        free(exe_path);
+        return 1;
     }
+    if (uv_exepath(exe_path, &path_size)) {
+        jl_error("fatal error: unexpected error while retrieving exepath");
+        free(exe_path);
+        return 1;
+    }
+
+    char* root_dir = dirname(dirname(exe_path));
+    char* depot_str = "JULIA_DEPOT_PATH=";
+    char* load_path_str = "JULIA_LOAD_PATH=";
+#ifdef _WIN32
+    char *julia_share_subdir = "\\share\\julia";
+#else
+    char *julia_share_subdir = "/share/julia";
+#endif
+    char *depot_path_env = calloc(sizeof(char), strlen(depot_str)    + strlen(root_dir) + strlen(julia_share_subdir) + 1);
+    char *load_path_env  = calloc(sizeof(char), strlen(load_path_str)+ strlen(root_dir) + strlen(julia_share_subdir) + 1);
+
+    strcat(depot_path_env, depot_str);
+    strcat(depot_path_env, root_dir);
+    strcat(depot_path_env, julia_share_subdir);
 
-    char buf[PATH_MAX];
-    snprintf(buf, sizeof(buf), "JULIA_DEPOT_PATH=%s/", dirname(dirname(free_path)));
-    putenv(buf);
-    putenv("JULIA_LOAD_PATH=@");
+    strcat(load_path_env, load_path_str);
+    strcat(load_path_env, root_dir);
+    strcat(load_path_env, julia_share_subdir); 
 
+    putenv(depot_path_env);
+    putenv(load_path_env);
     // JULIAC_PROGRAM_LIBNAME defined on command-line for compilation
     jl_options.image_file = JULIAC_PROGRAM_LIBNAME;
     julia_init(JL_IMAGE_JULIA_HOME);
@@ -85,6 +106,10 @@ int main(int argc, char *argv[])
     int retcode = julia_main(ARGS);
 
     // Cleanup and gracefully exit
+
+    free(depot_path_env);
+    free(load_path_env);
+    free(exe_path);
     jl_atexit_hook(retcode);
     return retcode;
 }

test/runtests.jl

@@ -100,6 +100,10 @@ end
             @test occursin("is_crayons_loaded() = true", app_output)
             # Check distributed
             @test occursin("n = 20000000", app_output)
+            @test occursin("From worker 2:    8", app_output)
+            @test occursin("From worker 3:    8", app_output)
+            @test occursin("From worker 4:    8", app_output)
+            @test occursin("From worker 5:    8", app_output)
         end
     end