From 5110121d87053110eed09186ab21791c70cc5db6 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sat, 22 Jan 2022 16:02:25 -0600 Subject: [PATCH 01/11] update version --- app_builder/lib/app_builder/macos.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app_builder/lib/app_builder/macos.ex b/app_builder/lib/app_builder/macos.ex index 8d840fd6531..c53b5f35fa8 100644 --- a/app_builder/lib/app_builder/macos.ex +++ b/app_builder/lib/app_builder/macos.ex @@ -244,9 +244,9 @@ defmodule AppBuilder.MacOS do CFBundleDisplayName <%= app_name %> CFBundleShortVersionString - <%= app_version %>on} + <%= app_version %> CFBundleVersion - <%= app_version %>on} + <%= app_version %> CFBundleIconFile AppIcon CFBundleIconName From 01cd52ae996eaa1d46cd0bab0bbfaac043b5efa0 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sun, 23 Jan 2022 14:38:39 -0600 Subject: [PATCH 02/11] It adds the bin directory to the environment path. --- app_builder/lib/app_builder/macos.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app_builder/lib/app_builder/macos.ex b/app_builder/lib/app_builder/macos.ex index c53b5f35fa8..bb5b0bc75b6 100644 --- a/app_builder/lib/app_builder/macos.ex +++ b/app_builder/lib/app_builder/macos.ex @@ -147,7 +147,16 @@ defmodule AppBuilder.MacOS do let releaseScriptPath = Bundle.main.path(forResource: "rel/bin/mac_app", ofType: "")! + let resourcePath = Bundle.main.resourcePath ?? "" + + var environment = ProcessInfo.processInfo.environment + let path = environment["PATH"] ?? "" + let elixirBinPath = "\\(resourcePath)/rel/elixir/bin" + + environment["PATH"] = "\\(elixirBinPath):\\(path)" + let task = Process() + task.environment = environment task.launchPath = releaseScriptPath task.arguments = ["start"] task.standardOutput = logFile From 611090860d9e7f48ee9081f312fecaacec7fdfc2 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sun, 23 Jan 2022 14:41:17 -0600 Subject: [PATCH 03/11] It adds elixir, erlang into the released app. --- app_builder/lib/app_builder.ex | 208 +++++++++++++++++++++++++++++++++ mix.exs | 14 ++- 2 files changed, 219 insertions(+), 3 deletions(-) diff --git a/app_builder/lib/app_builder.ex b/app_builder/lib/app_builder.ex index 129f536b6cb..6d0a8bf684d 100644 --- a/app_builder/lib/app_builder.ex +++ b/app_builder/lib/app_builder.ex @@ -1,5 +1,213 @@ defmodule AppBuilder do + require Logger + defdelegate build_mac_app(release, options), to: AppBuilder.MacOS defdelegate build_mac_app_dmg(release, options), to: AppBuilder.MacOS + + @doc """ + Copies ERTS into the release. + """ + @spec copy_erlang(Mix.Release.t()) :: Mix.Release.t() + def copy_erlang(release) do + {erts_source, erts_bin_dir, erts_lib_dir, erts_version} = erts_data() + + erts_destination_source = Path.join(release.path, "erts-#{erts_version}/bin") + File.mkdir_p!(erts_destination_source) + + erts_source + |> Path.join("bin") + |> File.cp_r!(erts_destination_source, fn _, _ -> false end) + + _ = File.rm(Path.join(erts_destination_source, "erl")) + _ = File.rm(Path.join(erts_destination_source, "erl.ini")) + + erts_destination_source + |> Path.join("erl") + |> File.write!(~S""" + #!/bin/sh + SELF=$(readlink "$0" || true) + if [ -z "$SELF" ]; then SELF="$0"; fi + BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)" + ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}" + EMU=beam + PROGNAME=$(echo "$0" | sed 's/.*\///') + export EMU + export ROOTDIR + export BINDIR + export PROGNAME + exec "$BINDIR/erlexec" ${1+"$@"} + """) + executable!(Path.join(erts_destination_source, "erl")) + + # Copy lib + erts_destination_lib = Path.join(release.path, "lib") + File.mkdir_p!(erts_destination_lib) + + erts_lib_dir + |> File.cp_r!(erts_destination_lib, fn _, _ -> false end) + + # copy start.boot to /rel/bin + erts_destination_bin = Path.join(release.path, "/bin") + start_boot_file = Path.join(erts_destination_bin, "start.boot") + File.mkdir_p!(erts_destination_bin) + + erts_bin_dir + |> Path.join("start.boot") + |> File.cp!(start_boot_file, fn _, _ -> false end) + + %{release | erts_source: erts_source} + end + + @erts_bin [~s[ERTS_BIN="$ERTS_BIN"], ~s[ERTS_BIN=!ERTS_BIN!]] + + defp replace_erts_bin(contents, release, new_path) do + if release.erts_source do + String.replace(contents, @erts_bin, ~s[ERTS_BIN=#{new_path}]) + else + contents + end + end + + @doc """ + Copies elixir into the release. + """ + @spec copy_elixir(Mix.Release.t()) :: Mix.Release.t() + def copy_elixir(release) do + include_executables_for = Keyword.get(release.options, :include_executables_for, [:unix]) + + elixir_bin_path = Application.app_dir(:elixir, "../../bin") + bin_path = Path.join(release.path, "bin") + File.mkdir_p!(bin_path) + + # download and unzip + standalone_destination = Path.join(release.path, "elixir") + download_elixir_at_destination(standalone_destination) + + # patch elixir file to look for the right erts /rel/releases/#{release.version}/elixir + patch_elixir(include_executables_for, release, + fn filename -> + Path.join(elixir_bin_path, filename) + end, + fn filename -> + Path.join(release.version_path, filename) + end + ) + + # patch elixir file to look for the right erts /rel/elixir/bin/elixir + patch_elixir(include_executables_for, release, + fn filename -> + Path.join([standalone_destination, "bin", filename]) + end, + fn filename -> + Path.join([standalone_destination, "bin", filename]) + end + ) + + release + end + + @elixir_standalone_version "1.13.2" + + defp download_elixir_at_destination(destination) do + url = "https://github.com/elixir-lang/elixir/releases/download/v#{@elixir_standalone_version}/Precompiled.zip" + binary = fetch_body!(url) + File.write!("/tmp/elixir_#{@elixir_standalone_version}.zip", binary, [:binary]) + :zip.unzip('/tmp/elixir_#{@elixir_standalone_version}.zip', cwd: destination) + end + + defp patch_elixir(include_executables_for, release, fn_source, fn_target) do + for os <- include_executables_for do + for {filename, contents_fun} <- elixir_cli_for(os, release) do + source = fn_source.(filename) + + if File.regular?(source) do + target = fn_target.(filename) + + File.write!(target, contents_fun.(source)) + executable!(target) + + else + skipping("#{filename} for #{os} (bin/#{filename} not found in the Elixir installation)") + end + end + end + end + + defp elixir_cli_for(:unix, release) do + [ + {"elixir", + &(&1 + |> File.read!() + |> String.replace(~s[ -pa "$SCRIPT_PATH"/../lib/*/ebin], "") + |> replace_erts_bin(release, ~s["$SCRIPT_PATH"/../../erts-#{release.erts_version}/bin/]))}, + {"iex", &File.read!/1} + ] + end + + defp elixir_cli_for(:windows, release) do + [ + {"elixir.bat", + &(&1 + |> File.read!() + |> String.replace(~s[goto expand_erl_libs], ~s[goto run]) + |> replace_erts_bin(release, ~s[%~dp0\\..\\..\\erts-#{release.erts_version}\\bin\\]))}, + {"iex.bat", &File.read!/1} + ] + end + + defp erts_data do + version = :erlang.system_info(:version) + {:filename.join(:code.root_dir(), 'erts-#{version}'), :filename.join(:code.root_dir(), 'bin'), :code.lib_dir(), version} + end + + defp fetch_body!(url) do + url = String.to_charlist(url) + Logger.debug("Downloading elixir from #{url}") + + {:ok, _} = Application.ensure_all_started(:inets) + {:ok, _} = Application.ensure_all_started(:ssl) + + if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do + Logger.debug("Using HTTP_PROXY: #{proxy}") + %{host: host, port: port} = URI.parse(proxy) + :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}]) + end + + if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do + Logger.debug("Using HTTPS_PROXY: #{proxy}") + %{host: host, port: port} = URI.parse(proxy) + :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}]) + end + + # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets + cacertfile = CAStore.file_path() |> String.to_charlist() + + http_options = [ + ssl: [ + verify: :verify_peer, + cacertfile: cacertfile, + depth: 2, + customize_hostname_check: [ + match_fun: :public_key.pkix_verify_hostname_match_fun(:https) + ] + ] + ] + + options = [body_format: :binary] + + case :httpc.request(:get, {url, []}, http_options, options) do + {:ok, {{_, 200, _}, _headers, body}} -> + body + + other -> + raise "couldn't fetch #{url}: #{inspect(other)}" + end + end + + defp skipping(message) do + Mix.shell().info([:yellow, "* skipping ", :reset, message]) + end + + defp executable!(path), do: File.chmod!(path, 0o755) end diff --git a/mix.exs b/mix.exs index 933c2e97527..0dd590e66e8 100644 --- a/mix.exs +++ b/mix.exs @@ -32,7 +32,7 @@ defmodule Livebook.MixProject do ] end - defp extra_applications(:app), do: [:wx] + defp extra_applications(:app), do: [:wx, :inets] defp extra_applications(_), do: [] defp elixirc_paths(:test), do: ["lib", "test/support"] @@ -115,13 +115,15 @@ defmodule Livebook.MixProject do ], mac_app: [ include_executables_for: [:unix], + include_erts: false, rel_templates_path: "rel/app", - steps: [:assemble, &build_mac_app/1] + steps: [:assemble, &standalone_erlang_elixir/1, &build_mac_app/1] ], mac_app_dmg: [ include_executables_for: [:unix], + include_erts: false, rel_templates_path: "rel/app", - steps: [:assemble, &build_mac_app_dmg/1] + steps: [:assemble, &standalone_erlang_elixir/1, &build_mac_app_dmg/1] ] ] end @@ -136,6 +138,12 @@ defmodule Livebook.MixProject do ] ] + defp standalone_erlang_elixir(release) do + release + |> AppBuilder.copy_erlang() + |> AppBuilder.copy_elixir() + end + defp build_mac_app(release) do AppBuilder.build_mac_app(release, @app_options) end From 602f6eec9daa30be65e440b62ba27c81c73e2e17 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sun, 23 Jan 2022 15:42:17 -0600 Subject: [PATCH 04/11] remove copy's functions --- app_builder/lib/app_builder.ex | 208 --------------------------------- 1 file changed, 208 deletions(-) diff --git a/app_builder/lib/app_builder.ex b/app_builder/lib/app_builder.ex index 6d0a8bf684d..129f536b6cb 100644 --- a/app_builder/lib/app_builder.ex +++ b/app_builder/lib/app_builder.ex @@ -1,213 +1,5 @@ defmodule AppBuilder do - require Logger - defdelegate build_mac_app(release, options), to: AppBuilder.MacOS defdelegate build_mac_app_dmg(release, options), to: AppBuilder.MacOS - - @doc """ - Copies ERTS into the release. - """ - @spec copy_erlang(Mix.Release.t()) :: Mix.Release.t() - def copy_erlang(release) do - {erts_source, erts_bin_dir, erts_lib_dir, erts_version} = erts_data() - - erts_destination_source = Path.join(release.path, "erts-#{erts_version}/bin") - File.mkdir_p!(erts_destination_source) - - erts_source - |> Path.join("bin") - |> File.cp_r!(erts_destination_source, fn _, _ -> false end) - - _ = File.rm(Path.join(erts_destination_source, "erl")) - _ = File.rm(Path.join(erts_destination_source, "erl.ini")) - - erts_destination_source - |> Path.join("erl") - |> File.write!(~S""" - #!/bin/sh - SELF=$(readlink "$0" || true) - if [ -z "$SELF" ]; then SELF="$0"; fi - BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)" - ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}" - EMU=beam - PROGNAME=$(echo "$0" | sed 's/.*\///') - export EMU - export ROOTDIR - export BINDIR - export PROGNAME - exec "$BINDIR/erlexec" ${1+"$@"} - """) - executable!(Path.join(erts_destination_source, "erl")) - - # Copy lib - erts_destination_lib = Path.join(release.path, "lib") - File.mkdir_p!(erts_destination_lib) - - erts_lib_dir - |> File.cp_r!(erts_destination_lib, fn _, _ -> false end) - - # copy start.boot to /rel/bin - erts_destination_bin = Path.join(release.path, "/bin") - start_boot_file = Path.join(erts_destination_bin, "start.boot") - File.mkdir_p!(erts_destination_bin) - - erts_bin_dir - |> Path.join("start.boot") - |> File.cp!(start_boot_file, fn _, _ -> false end) - - %{release | erts_source: erts_source} - end - - @erts_bin [~s[ERTS_BIN="$ERTS_BIN"], ~s[ERTS_BIN=!ERTS_BIN!]] - - defp replace_erts_bin(contents, release, new_path) do - if release.erts_source do - String.replace(contents, @erts_bin, ~s[ERTS_BIN=#{new_path}]) - else - contents - end - end - - @doc """ - Copies elixir into the release. - """ - @spec copy_elixir(Mix.Release.t()) :: Mix.Release.t() - def copy_elixir(release) do - include_executables_for = Keyword.get(release.options, :include_executables_for, [:unix]) - - elixir_bin_path = Application.app_dir(:elixir, "../../bin") - bin_path = Path.join(release.path, "bin") - File.mkdir_p!(bin_path) - - # download and unzip - standalone_destination = Path.join(release.path, "elixir") - download_elixir_at_destination(standalone_destination) - - # patch elixir file to look for the right erts /rel/releases/#{release.version}/elixir - patch_elixir(include_executables_for, release, - fn filename -> - Path.join(elixir_bin_path, filename) - end, - fn filename -> - Path.join(release.version_path, filename) - end - ) - - # patch elixir file to look for the right erts /rel/elixir/bin/elixir - patch_elixir(include_executables_for, release, - fn filename -> - Path.join([standalone_destination, "bin", filename]) - end, - fn filename -> - Path.join([standalone_destination, "bin", filename]) - end - ) - - release - end - - @elixir_standalone_version "1.13.2" - - defp download_elixir_at_destination(destination) do - url = "https://github.com/elixir-lang/elixir/releases/download/v#{@elixir_standalone_version}/Precompiled.zip" - binary = fetch_body!(url) - File.write!("/tmp/elixir_#{@elixir_standalone_version}.zip", binary, [:binary]) - :zip.unzip('/tmp/elixir_#{@elixir_standalone_version}.zip', cwd: destination) - end - - defp patch_elixir(include_executables_for, release, fn_source, fn_target) do - for os <- include_executables_for do - for {filename, contents_fun} <- elixir_cli_for(os, release) do - source = fn_source.(filename) - - if File.regular?(source) do - target = fn_target.(filename) - - File.write!(target, contents_fun.(source)) - executable!(target) - - else - skipping("#{filename} for #{os} (bin/#{filename} not found in the Elixir installation)") - end - end - end - end - - defp elixir_cli_for(:unix, release) do - [ - {"elixir", - &(&1 - |> File.read!() - |> String.replace(~s[ -pa "$SCRIPT_PATH"/../lib/*/ebin], "") - |> replace_erts_bin(release, ~s["$SCRIPT_PATH"/../../erts-#{release.erts_version}/bin/]))}, - {"iex", &File.read!/1} - ] - end - - defp elixir_cli_for(:windows, release) do - [ - {"elixir.bat", - &(&1 - |> File.read!() - |> String.replace(~s[goto expand_erl_libs], ~s[goto run]) - |> replace_erts_bin(release, ~s[%~dp0\\..\\..\\erts-#{release.erts_version}\\bin\\]))}, - {"iex.bat", &File.read!/1} - ] - end - - defp erts_data do - version = :erlang.system_info(:version) - {:filename.join(:code.root_dir(), 'erts-#{version}'), :filename.join(:code.root_dir(), 'bin'), :code.lib_dir(), version} - end - - defp fetch_body!(url) do - url = String.to_charlist(url) - Logger.debug("Downloading elixir from #{url}") - - {:ok, _} = Application.ensure_all_started(:inets) - {:ok, _} = Application.ensure_all_started(:ssl) - - if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do - Logger.debug("Using HTTP_PROXY: #{proxy}") - %{host: host, port: port} = URI.parse(proxy) - :httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}]) - end - - if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do - Logger.debug("Using HTTPS_PROXY: #{proxy}") - %{host: host, port: port} = URI.parse(proxy) - :httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}]) - end - - # https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets - cacertfile = CAStore.file_path() |> String.to_charlist() - - http_options = [ - ssl: [ - verify: :verify_peer, - cacertfile: cacertfile, - depth: 2, - customize_hostname_check: [ - match_fun: :public_key.pkix_verify_hostname_match_fun(:https) - ] - ] - ] - - options = [body_format: :binary] - - case :httpc.request(:get, {url, []}, http_options, options) do - {:ok, {{_, 200, _}, _headers, body}} -> - body - - other -> - raise "couldn't fetch #{url}: #{inspect(other)}" - end - end - - defp skipping(message) do - Mix.shell().info([:yellow, "* skipping ", :reset, message]) - end - - defp executable!(path), do: File.chmod!(path, 0o755) end From 6ea2ce91741e9ab744e2d0be4787c997a96038f7 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sun, 23 Jan 2022 16:29:31 -0600 Subject: [PATCH 05/11] moved copy code to standalone module --- mix.exs | 8 +- rel/app/standalone.exs | 176 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 rel/app/standalone.exs diff --git a/mix.exs b/mix.exs index 0dd590e66e8..17f0a075313 100644 --- a/mix.exs +++ b/mix.exs @@ -32,7 +32,7 @@ defmodule Livebook.MixProject do ] end - defp extra_applications(:app), do: [:wx, :inets] + defp extra_applications(:app), do: [:wx] defp extra_applications(_), do: [] defp elixirc_paths(:test), do: ["lib", "test/support"] @@ -139,9 +139,11 @@ defmodule Livebook.MixProject do ] defp standalone_erlang_elixir(release) do + Code.require_file("rel/app/standalone.exs") + release - |> AppBuilder.copy_erlang() - |> AppBuilder.copy_elixir() + |> Standalone.copy_erlang() + |> Standalone.copy_elixir("1.13.2") end defp build_mac_app(release) do diff --git a/rel/app/standalone.exs b/rel/app/standalone.exs new file mode 100644 index 00000000000..21029426868 --- /dev/null +++ b/rel/app/standalone.exs @@ -0,0 +1,176 @@ +defmodule Standalone do + @moduledoc false + require Logger + + @doc """ + Copies ERTS into the release. + """ + @spec copy_erlang(Mix.Release.t()) :: Mix.Release.t() + def copy_erlang(release) do + {erts_source, erts_bin_dir, erts_lib_dir, erts_version} = erts_data() + + erts_destination_source = Path.join(release.path, "erts-#{erts_version}/bin") + File.mkdir_p!(erts_destination_source) + + erts_source + |> Path.join("bin") + |> File.cp_r!(erts_destination_source, fn _, _ -> false end) + + _ = File.rm(Path.join(erts_destination_source, "erl")) + _ = File.rm(Path.join(erts_destination_source, "erl.ini")) + + erts_destination_source + |> Path.join("erl") + |> File.write!(~S""" + #!/bin/sh + SELF=$(readlink "$0" || true) + if [ -z "$SELF" ]; then SELF="$0"; fi + BINDIR="$(cd "$(dirname "$SELF")" && pwd -P)" + ROOTDIR="${ERL_ROOTDIR:-"$(dirname "$(dirname "$BINDIR")")"}" + EMU=beam + PROGNAME=$(echo "$0" | sed 's/.*\///') + export EMU + export ROOTDIR + export BINDIR + export PROGNAME + exec "$BINDIR/erlexec" ${1+"$@"} + """) + executable!(Path.join(erts_destination_source, "erl")) + + # Copy lib + erts_destination_lib = Path.join(release.path, "lib") + File.mkdir_p!(erts_destination_lib) + + erts_lib_dir + |> File.cp_r!(erts_destination_lib, fn _, _ -> false end) + + # copy start.boot to /rel/bin + erts_destination_bin = Path.join(release.path, "/bin") + start_boot_file = Path.join(erts_destination_bin, "start.boot") + File.mkdir_p!(erts_destination_bin) + + erts_bin_dir + |> Path.join("start.boot") + |> File.cp!(start_boot_file, fn _, _ -> false end) + + %{release | erts_source: erts_source} + end + + @erts_bin [~s[ERTS_BIN="$ERTS_BIN"], ~s[ERTS_BIN=!ERTS_BIN!]] + + defp replace_erts_bin(contents, release, new_path) do + if release.erts_source do + String.replace(contents, @erts_bin, ~s[ERTS_BIN=#{new_path}]) + else + contents + end + end + + @doc """ + Copies elixir into the release. + """ + @spec copy_elixir(Mix.Release.t(), elixir_version :: String.t()) :: Mix.Release.t() + def copy_elixir(release, elixir_version) do + include_executables_for = Keyword.get(release.options, :include_executables_for, [:unix]) + + elixir_bin_path = Application.app_dir(:elixir, "../../bin") + bin_path = Path.join(release.path, "bin") + File.mkdir_p!(bin_path) + + # download and unzip + standalone_destination = Path.join(release.path, "elixir") + download_elixir_at_destination(standalone_destination, elixir_version) + + # patch elixir file to look for the right erts /rel/releases/#{release.version}/elixir + patch_elixir(include_executables_for, release, + fn filename -> + Path.join(elixir_bin_path, filename) + end, + fn filename -> + Path.join(release.version_path, filename) + end + ) + + # patch elixir file to look for the right erts /rel/elixir/bin/elixir + patch_elixir(include_executables_for, release, + fn filename -> + Path.join([standalone_destination, "bin", filename]) + end, + fn filename -> + Path.join([standalone_destination, "bin", filename]) + end + ) + + release + end + + defp download_elixir_at_destination(destination, elixir_version) do + url = "https://github.com/elixir-lang/elixir/releases/download/v#{elixir_version}/Precompiled.zip" + binary = fetch_body!(url) + File.write!("/tmp/elixir_#{elixir_version}.zip", binary, [:binary]) + :zip.unzip('/tmp/elixir_#{elixir_version}.zip', cwd: destination) + end + + defp patch_elixir(include_executables_for, release, fn_source, fn_target) do + for os <- include_executables_for do + for {filename, contents_fun} <- elixir_cli_for(os, release) do + source = fn_source.(filename) + + if File.regular?(source) do + target = fn_target.(filename) + + File.write!(target, contents_fun.(source)) + executable!(target) + + else + skipping("#{filename} for #{os} (bin/#{filename} not found in the Elixir installation)") + end + end + end + end + + defp elixir_cli_for(:unix, release) do + [ + {"elixir", + &(&1 + |> File.read!() + |> String.replace(~s[ -pa "$SCRIPT_PATH"/../lib/*/ebin], "") + |> replace_erts_bin(release, ~s["$SCRIPT_PATH"/../../erts-#{release.erts_version}/bin/]))}, + {"iex", &File.read!/1} + ] + end + + defp elixir_cli_for(:windows, release) do + [ + {"elixir.bat", + &(&1 + |> File.read!() + |> String.replace(~s[goto expand_erl_libs], ~s[goto run]) + |> replace_erts_bin(release, ~s[%~dp0\\..\\..\\erts-#{release.erts_version}\\bin\\]))}, + {"iex.bat", &File.read!/1} + ] + end + + defp erts_data do + version = :erlang.system_info(:version) + {:filename.join(:code.root_dir(), 'erts-#{version}'), :filename.join(:code.root_dir(), 'bin'), :code.lib_dir(), version} + end + + defp fetch_body!(url) do + Logger.debug("Downloading elixir from #{url}") + case Livebook.Utils.HTTP.request(:get, url, [timeout: 15_000]) do + {:ok, 200, _headers, body} -> + body + + {:error, error} -> + raise "couldn't fetch #{url}: #{inspect(error)}" + end + end + + defp skipping(message) do + Mix.shell().info([:yellow, "* skipping ", :reset, message]) + end + + defp executable!(path), do: File.chmod!(path, 0o755) + +end From 76731dadda52e358daa1df3c7693e1584ced9016 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sun, 23 Jan 2022 16:31:41 -0600 Subject: [PATCH 06/11] keeps a generic name, and allow additional paths --- app_builder/lib/app_builder/macos.ex | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app_builder/lib/app_builder/macos.ex b/app_builder/lib/app_builder/macos.ex index bb5b0bc75b6..f27dc0229b6 100644 --- a/app_builder/lib/app_builder/macos.ex +++ b/app_builder/lib/app_builder/macos.ex @@ -109,7 +109,7 @@ defmodule AppBuilder.MacOS do File.mkdir_p!("tmp") launcher_src_path = "tmp/Launcher.swift" - File.write!(launcher_src_path, launcher()) + File.write!(launcher_src_path, launcher(["/rel/elixir/bin"])) launcher_path = Path.join([app_bundle_path, "Contents", "MacOS", app_name <> "Launcher"]) File.mkdir_p!(Path.dirname(launcher_path)) @@ -131,7 +131,11 @@ defmodule AppBuilder.MacOS do release end - defp launcher do + defp launcher(additional_paths) do + additional_paths = additional_paths + |> Enum.map(&("\\(resourcePath)#{&1}")) + |> Enum.join(":") + """ import Foundation import Cocoa @@ -148,12 +152,12 @@ defmodule AppBuilder.MacOS do let releaseScriptPath = Bundle.main.path(forResource: "rel/bin/mac_app", ofType: "")! let resourcePath = Bundle.main.resourcePath ?? "" + let additionalPaths = "#{additional_paths}" var environment = ProcessInfo.processInfo.environment let path = environment["PATH"] ?? "" - let elixirBinPath = "\\(resourcePath)/rel/elixir/bin" - environment["PATH"] = "\\(elixirBinPath):\\(path)" + environment["PATH"] = "\\(additionalPaths):\\(path)" let task = Process() task.environment = environment From 676e4ba52a0dd3a17b76b638324787786553c5db Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Sun, 23 Jan 2022 17:01:52 -0600 Subject: [PATCH 07/11] copy to the vendor directory --- app_builder/lib/app_builder/macos.ex | 2 +- rel/app/standalone.exs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app_builder/lib/app_builder/macos.ex b/app_builder/lib/app_builder/macos.ex index f27dc0229b6..0381659c0c6 100644 --- a/app_builder/lib/app_builder/macos.ex +++ b/app_builder/lib/app_builder/macos.ex @@ -109,7 +109,7 @@ defmodule AppBuilder.MacOS do File.mkdir_p!("tmp") launcher_src_path = "tmp/Launcher.swift" - File.write!(launcher_src_path, launcher(["/rel/elixir/bin"])) + File.write!(launcher_src_path, launcher(["/rel/vendor/bin"])) launcher_path = Path.join([app_bundle_path, "Contents", "MacOS", app_name <> "Launcher"]) File.mkdir_p!(Path.dirname(launcher_path)) diff --git a/rel/app/standalone.exs b/rel/app/standalone.exs index 21029426868..1c713475f50 100644 --- a/rel/app/standalone.exs +++ b/rel/app/standalone.exs @@ -78,7 +78,7 @@ defmodule Standalone do File.mkdir_p!(bin_path) # download and unzip - standalone_destination = Path.join(release.path, "elixir") + standalone_destination = Path.join(release.path, "vendor") download_elixir_at_destination(standalone_destination, elixir_version) # patch elixir file to look for the right erts /rel/releases/#{release.version}/elixir @@ -91,7 +91,7 @@ defmodule Standalone do end ) - # patch elixir file to look for the right erts /rel/elixir/bin/elixir + # patch elixir file to look for the right erts /rel/vendor/bin/elixir patch_elixir(include_executables_for, release, fn filename -> Path.join([standalone_destination, "bin", filename]) From 5ad49c0547fe094d299c71c4d3c60209e3ec70fd Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Mon, 24 Jan 2022 10:33:55 -0600 Subject: [PATCH 08/11] improve code for additional paths --- app_builder/lib/app_builder/macos.ex | 6 ++++-- mix.exs | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app_builder/lib/app_builder/macos.ex b/app_builder/lib/app_builder/macos.ex index 0381659c0c6..5a9e0c267c7 100644 --- a/app_builder/lib/app_builder/macos.ex +++ b/app_builder/lib/app_builder/macos.ex @@ -97,10 +97,12 @@ defmodule AppBuilder.MacOS do :logo_path, :info_plist, :url_schemes, - :document_types + :document_types, + :additional_paths ]) app_name = Keyword.fetch!(options, :name) + additional_paths = Keyword.get(options, :additional_paths, []) app_bundle_path = Path.join([Mix.Project.build_path(), "rel", "#{app_name}.app"]) File.rm_rf!(app_bundle_path) @@ -109,7 +111,7 @@ defmodule AppBuilder.MacOS do File.mkdir_p!("tmp") launcher_src_path = "tmp/Launcher.swift" - File.write!(launcher_src_path, launcher(["/rel/vendor/bin"])) + File.write!(launcher_src_path, launcher(additional_paths)) launcher_path = Path.join([app_bundle_path, "Contents", "MacOS", app_name <> "Launcher"]) File.mkdir_p!(Path.dirname(launcher_path)) diff --git a/mix.exs b/mix.exs index 17f0a075313..099e2d5fe8a 100644 --- a/mix.exs +++ b/mix.exs @@ -133,6 +133,7 @@ defmodule Livebook.MixProject do version: @version, logo_path: "static/images/logo.png", url_schemes: ["livebook"], + additional_paths: ["/rel/vendor/erts/bin", "/rel/vendor/elixir/bin"], document_types: [ %{name: "LiveMarkdown", role: "Editor", extensions: ["livemd"]} ] From 112c4c2ffa9d006d1a87032682a209dabc0573ed Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Mon, 24 Jan 2022 10:34:24 -0600 Subject: [PATCH 09/11] move erts to vendor and update file patches --- rel/app/standalone.exs | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/rel/app/standalone.exs b/rel/app/standalone.exs index 1c713475f50..e20c41e655d 100644 --- a/rel/app/standalone.exs +++ b/rel/app/standalone.exs @@ -7,9 +7,9 @@ defmodule Standalone do """ @spec copy_erlang(Mix.Release.t()) :: Mix.Release.t() def copy_erlang(release) do - {erts_source, erts_bin_dir, erts_lib_dir, erts_version} = erts_data() + {erts_source, erts_bin_dir, erts_lib_dir, _erts_version} = erts_data() - erts_destination_source = Path.join(release.path, "erts-#{erts_version}/bin") + erts_destination_source = Path.join(release.path, "vendor/erts/bin") File.mkdir_p!(erts_destination_source) erts_source @@ -38,14 +38,14 @@ defmodule Standalone do executable!(Path.join(erts_destination_source, "erl")) # Copy lib - erts_destination_lib = Path.join(release.path, "lib") + erts_destination_lib = Path.join(release.path, "/vendor/lib") File.mkdir_p!(erts_destination_lib) erts_lib_dir |> File.cp_r!(erts_destination_lib, fn _, _ -> false end) # copy start.boot to /rel/bin - erts_destination_bin = Path.join(release.path, "/bin") + erts_destination_bin = Path.join(release.path, "/vendor/bin") start_boot_file = Path.join(erts_destination_bin, "start.boot") File.mkdir_p!(erts_destination_bin) @@ -73,24 +73,10 @@ defmodule Standalone do def copy_elixir(release, elixir_version) do include_executables_for = Keyword.get(release.options, :include_executables_for, [:unix]) - elixir_bin_path = Application.app_dir(:elixir, "../../bin") - bin_path = Path.join(release.path, "bin") - File.mkdir_p!(bin_path) - # download and unzip - standalone_destination = Path.join(release.path, "vendor") + standalone_destination = Path.join(release.path, "vendor/elixir") download_elixir_at_destination(standalone_destination, elixir_version) - # patch elixir file to look for the right erts /rel/releases/#{release.version}/elixir - patch_elixir(include_executables_for, release, - fn filename -> - Path.join(elixir_bin_path, filename) - end, - fn filename -> - Path.join(release.version_path, filename) - end - ) - # patch elixir file to look for the right erts /rel/vendor/bin/elixir patch_elixir(include_executables_for, release, fn filename -> @@ -134,8 +120,7 @@ defmodule Standalone do {"elixir", &(&1 |> File.read!() - |> String.replace(~s[ -pa "$SCRIPT_PATH"/../lib/*/ebin], "") - |> replace_erts_bin(release, ~s["$SCRIPT_PATH"/../../erts-#{release.erts_version}/bin/]))}, + |> replace_erts_bin(release, ~s["$SCRIPT_PATH"/../../erts/bin/]))}, {"iex", &File.read!/1} ] end @@ -145,8 +130,7 @@ defmodule Standalone do {"elixir.bat", &(&1 |> File.read!() - |> String.replace(~s[goto expand_erl_libs], ~s[goto run]) - |> replace_erts_bin(release, ~s[%~dp0\\..\\..\\erts-#{release.erts_version}\\bin\\]))}, + |> replace_erts_bin(release, ~s[%~dp0\\..\\..\\erts\\bin\\]))}, {"iex.bat", &File.read!/1} ] end @@ -158,7 +142,7 @@ defmodule Standalone do defp fetch_body!(url) do Logger.debug("Downloading elixir from #{url}") - case Livebook.Utils.HTTP.request(:get, url, [timeout: 15_000]) do + case Livebook.Utils.HTTP.request(:get, url, [timeout: :infinity]) do {:ok, 200, _headers, body} -> body From 5e88d93d2cd4dddf5ec8dc3f58ebf0cfdc197861 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Mon, 24 Jan 2022 12:49:52 -0600 Subject: [PATCH 10/11] add missing boot files, and remove patching --- rel/app/standalone.exs | 76 +++++------------------------------------- 1 file changed, 9 insertions(+), 67 deletions(-) diff --git a/rel/app/standalone.exs b/rel/app/standalone.exs index e20c41e655d..6bf86d37150 100644 --- a/rel/app/standalone.exs +++ b/rel/app/standalone.exs @@ -46,24 +46,16 @@ defmodule Standalone do # copy start.boot to /rel/bin erts_destination_bin = Path.join(release.path, "/vendor/bin") - start_boot_file = Path.join(erts_destination_bin, "start.boot") - File.mkdir_p!(erts_destination_bin) - erts_bin_dir - |> Path.join("start.boot") - |> File.cp!(start_boot_file, fn _, _ -> false end) + boot_files = erts_bin_dir |> Path.join("*.boot") |> Path.wildcard() |> Enum.map(& String.split(&1, "/") |> List.last()) - %{release | erts_source: erts_source} - end - - @erts_bin [~s[ERTS_BIN="$ERTS_BIN"], ~s[ERTS_BIN=!ERTS_BIN!]] + File.mkdir_p!(erts_destination_bin) - defp replace_erts_bin(contents, release, new_path) do - if release.erts_source do - String.replace(contents, @erts_bin, ~s[ERTS_BIN=#{new_path}]) - else - contents + for boot_file <- boot_files do + erts_bin_dir |> Path.join(boot_file) |> File.cp!(Path.join(erts_destination_bin, boot_file)) end + + %{release | erts_source: erts_source} end @doc """ @@ -71,21 +63,13 @@ defmodule Standalone do """ @spec copy_elixir(Mix.Release.t(), elixir_version :: String.t()) :: Mix.Release.t() def copy_elixir(release, elixir_version) do - include_executables_for = Keyword.get(release.options, :include_executables_for, [:unix]) - # download and unzip standalone_destination = Path.join(release.path, "vendor/elixir") download_elixir_at_destination(standalone_destination, elixir_version) - # patch elixir file to look for the right erts /rel/vendor/bin/elixir - patch_elixir(include_executables_for, release, - fn filename -> - Path.join([standalone_destination, "bin", filename]) - end, - fn filename -> - Path.join([standalone_destination, "bin", filename]) - end - ) + # make executable + ["elixir", "elixirc", "mix", "iex"] + |> Enum.map(&(executable!(Path.join(standalone_destination, "bin/#{&1}")))) release end @@ -97,44 +81,6 @@ defmodule Standalone do :zip.unzip('/tmp/elixir_#{elixir_version}.zip', cwd: destination) end - defp patch_elixir(include_executables_for, release, fn_source, fn_target) do - for os <- include_executables_for do - for {filename, contents_fun} <- elixir_cli_for(os, release) do - source = fn_source.(filename) - - if File.regular?(source) do - target = fn_target.(filename) - - File.write!(target, contents_fun.(source)) - executable!(target) - - else - skipping("#{filename} for #{os} (bin/#{filename} not found in the Elixir installation)") - end - end - end - end - - defp elixir_cli_for(:unix, release) do - [ - {"elixir", - &(&1 - |> File.read!() - |> replace_erts_bin(release, ~s["$SCRIPT_PATH"/../../erts/bin/]))}, - {"iex", &File.read!/1} - ] - end - - defp elixir_cli_for(:windows, release) do - [ - {"elixir.bat", - &(&1 - |> File.read!() - |> replace_erts_bin(release, ~s[%~dp0\\..\\..\\erts\\bin\\]))}, - {"iex.bat", &File.read!/1} - ] - end - defp erts_data do version = :erlang.system_info(:version) {:filename.join(:code.root_dir(), 'erts-#{version}'), :filename.join(:code.root_dir(), 'bin'), :code.lib_dir(), version} @@ -151,10 +97,6 @@ defmodule Standalone do end end - defp skipping(message) do - Mix.shell().info([:yellow, "* skipping ", :reset, message]) - end - defp executable!(path), do: File.chmod!(path, 0o755) end From 03add6de10853138553556b7961901451bbfb858 Mon Sep 17 00:00:00 2001 From: Roberto Estrada Date: Mon, 24 Jan 2022 13:19:38 -0600 Subject: [PATCH 11/11] move erlang files to match with vendor/bin --- mix.exs | 2 +- rel/app/standalone.exs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index 099e2d5fe8a..1489990d3ea 100644 --- a/mix.exs +++ b/mix.exs @@ -133,7 +133,7 @@ defmodule Livebook.MixProject do version: @version, logo_path: "static/images/logo.png", url_schemes: ["livebook"], - additional_paths: ["/rel/vendor/erts/bin", "/rel/vendor/elixir/bin"], + additional_paths: ["/rel/vendor/bin", "/rel/vendor/elixir/bin"], document_types: [ %{name: "LiveMarkdown", role: "Editor", extensions: ["livemd"]} ] diff --git a/rel/app/standalone.exs b/rel/app/standalone.exs index 6bf86d37150..f2d26aceec0 100644 --- a/rel/app/standalone.exs +++ b/rel/app/standalone.exs @@ -9,7 +9,7 @@ defmodule Standalone do def copy_erlang(release) do {erts_source, erts_bin_dir, erts_lib_dir, _erts_version} = erts_data() - erts_destination_source = Path.join(release.path, "vendor/erts/bin") + erts_destination_source = Path.join(release.path, "vendor/bin") File.mkdir_p!(erts_destination_source) erts_source @@ -38,14 +38,14 @@ defmodule Standalone do executable!(Path.join(erts_destination_source, "erl")) # Copy lib - erts_destination_lib = Path.join(release.path, "/vendor/lib") + erts_destination_lib = Path.join(release.path, "lib") File.mkdir_p!(erts_destination_lib) erts_lib_dir |> File.cp_r!(erts_destination_lib, fn _, _ -> false end) - # copy start.boot to /rel/bin - erts_destination_bin = Path.join(release.path, "/vendor/bin") + # copy *.boot files to /bin + erts_destination_bin = Path.join(release.path, "bin") boot_files = erts_bin_dir |> Path.join("*.boot") |> Path.wildcard() |> Enum.map(& String.split(&1, "/") |> List.last())