Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add a way to run custom scripts for environment setup #1315

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions azure-pipelines/e2e-ports/env-scripts/env-cmake-script.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

file(WRITE "${BUILDTREES_DIR}/env-script.log" "DOWNLOADS:${DOWNLOADS}")
15 changes: 15 additions & 0 deletions azure-pipelines/e2e-ports/env-scripts/env-scripts.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
set(VCPKG_TARGET_ARCHITECTURE x64)
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE dynamic)
if(APPLE)
set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
elseif(UNIX)
set(VCPKG_CMAKE_SYSTEM_NAME Linux)
endif()

set(script_ext ".bat" )
if(UNIX)
set(script_ext "" )
endif()
set(VCPKG_ENVIRONMENT_SETUP_SCRIPTS "${CMAKE_CURRENT_LIST_DIR}/env-set-script${script_ext}")
list(APPEND VCPKG_ENVIRONMENT_SETUP_SCRIPTS "${CMAKE_CURRENT_LIST_DIR}/env-cmake-script.cmake")
4 changes: 4 additions & 0 deletions azure-pipelines/e2e-ports/env-scripts/env-set-script
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/usr/bin/bash

Neumann-A marked this conversation as resolved.
Show resolved Hide resolved
export VCPKG_ENV_TEST=TRUE
export VCPKG_ENV_TEST2=MORE_TESTING
2 changes: 2 additions & 0 deletions azure-pipelines/e2e-ports/env-scripts/env-set-script.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
set VCPKG_ENV_TEST=TRUE
set VCPKG_ENV_TEST2=MORE_TESTING
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

if(NOT DEFINED ENV{VCPKG_ENV_TEST} AND NOT "$ENV{VCPKG_ENV_TEST}" STREQUAL "TRUE")
message(FATAL_ERROR "ENV{VCPKG_ENV_TEST} not set or has wrong value of '$ENV{VCPKG_ENV_TEST}' (expected TRUE)")
endif()

if(NOT DEFINED ENV{VCPKG_ENV_TEST2} AND NOT "$ENV{VCPKG_ENV_TEST2}" STREQUAL "MORE_TESTING")
message(FATAL_ERROR "ENV{VCPKG_ENV_TEST2} not set or has wrong value of '$ENV{VCPKG_ENV_TEST2}' (expected MORE_TESTING)")
endif()

file(READ "${CURRENT_BUILDTREES_DIR}/../env-script.log" contents)
file(REMOVE "${CURRENT_BUILDTREES_DIR}/../env-script.log")

if(NOT contents STREQUAL "DOWNLOADS:${DOWNLOADS}")
message(FATAL_ERROR "contents (${contents}) of 'env-script.log' are not equal to 'DOWNLOADS:${DOWNLOADS}'")
endif()
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"name": "test-env-scripts",
"version": "0",
"description": "A port that tests VCPKG_ENVIRONMENT_SETUP_SCRIPTS"
}
4 changes: 4 additions & 0 deletions azure-pipelines/end-to-end-tests-dir/env-scripts.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
. $PSScriptRoot/../end-to-end-tests-prelude.ps1

Run-Vcpkg @directoryArgs "--overlay-triplets=$PSScriptRoot/../e2e-ports/env-scripts" "--overlay-ports=$PSScriptRoot/../e2e-ports/env-scripts" x-set-installed test-env-scripts --triplet env-scripts --binarysource=clear --debug
Throw-IfFailed
1 change: 1 addition & 0 deletions include/vcpkg/base/contractual-constants.h
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,7 @@ namespace vcpkg
inline constexpr StringLiteral CMakeVariableEditable = "_VCPKG_EDITABLE";
inline constexpr StringLiteral CMakeVariableEnvPassthrough = "VCPKG_ENV_PASSTHROUGH";
inline constexpr StringLiteral CMakeVariableEnvPassthroughUntracked = "VCPKG_ENV_PASSTHROUGH_UNTRACKED";
inline constexpr StringLiteral CMakeVariableEnvSetupScripts = "VCPKG_ENVIRONMENT_SETUP_SCRIPTS";
inline constexpr StringLiteral CMakeVariableFeatures = "FEATURES";
inline constexpr StringLiteral CMakeVariableFilename = "FILENAME";
inline constexpr StringLiteral CMakeVariableGit = "GIT";
Expand Down
16 changes: 12 additions & 4 deletions include/vcpkg/base/message-data.inc.h
Original file line number Diff line number Diff line change
Expand Up @@ -1840,6 +1840,11 @@ DECLARE_MESSAGE(InvalidCommentStyle,
"comments.")
DECLARE_MESSAGE(InvalidCommitId, (msg::commit_sha), "", "Invalid commit id: {commit_sha}")
DECLARE_MESSAGE(InvalidDefaultFeatureName, (), "", "'default' is a reserved feature name")
DECLARE_MESSAGE(InvalidEnvSetupScripts,
(msg::path),
"",
"Variable VCPKG_ENVIRONMENT_SETUP_SCRIPTS contains invalid file path: '{path}'. The value must be "
"an absolute path to an existent file.")
DECLARE_MESSAGE(InvalidFeature,
(),
"",
Expand Down Expand Up @@ -2829,11 +2834,14 @@ DECLARE_MESSAGE(VcpkgRegistriesCacheIsNotDirectory,
DECLARE_MESSAGE(VcpkgRootRequired, (), "", "Setting VCPKG_ROOT is required for standalone bootstrap.")
DECLARE_MESSAGE(VcpkgRootsDir, (msg::env_var), "", "The vcpkg root directory (default: {env_var})")
DECLARE_MESSAGE(VcpkgSendMetricsButDisabled, (), "", "passed --sendmetrics, but metrics are disabled.")
DECLARE_MESSAGE(VcvarsRunFailed, (), "", "failed to run vcvarsall.bat to get a Visual Studio environment")
DECLARE_MESSAGE(VcvarsRunFailedExitCode,
(msg::exit_code),
DECLARE_MESSAGE(CaptureCmdRunFailed,
(msg::command_name, msg::error_msg),
"",
"failed to run '{command_name}' to get environment ({error_msg}).")
DECLARE_MESSAGE(CaptureCmdEnvFailedExitCode,
(msg::command_name, msg::exit_code),
"",
"while trying to get a Visual Studio environment, vcvarsall.bat returned {exit_code}")
"while trying to capture an environment, '{command_name}' returned {exit_code}.")
DECLARE_MESSAGE(VersionBaselineMismatch,
(msg::expected, msg::actual, msg::package_name),
"{expected} and {actual} are versions",
Expand Down
2 changes: 0 additions & 2 deletions include/vcpkg/base/system.process.h
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,7 @@ namespace vcpkg
ExpectedL<int> cmd_execute(const Command& cmd);
ExpectedL<int> cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings);

#if defined(_WIN32)
Environment cmd_execute_and_capture_environment(const Command& cmd, const Environment& env);
#endif

void cmd_execute_background(const Command& cmd_line);

Expand Down
5 changes: 3 additions & 2 deletions include/vcpkg/commands.build.h
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,9 @@ namespace vcpkg
Optional<std::string> public_abi_override;
std::vector<std::string> passthrough_env_vars;
std::vector<std::string> passthrough_env_vars_tracked;
std::vector<std::string> environment_setup_scripts;
std::vector<Path> hash_additional_files;

Neumann-A marked this conversation as resolved.
Show resolved Hide resolved
Optional<Path> gamedk_latest_path;

Path toolchain_file() const;
Expand All @@ -143,6 +145,7 @@ namespace vcpkg
};

vcpkg::Command make_build_env_cmd(const PreBuildInfo& pre_build_info, const Toolset& toolset);
vcpkg::Command make_setup_env_cmd(const VcpkgPaths& paths, const Path& script);

struct ExtendedBuildResult
{
Expand Down Expand Up @@ -284,15 +287,13 @@ namespace vcpkg

const TripletMapEntry& get_triplet_cache(const ReadOnlyFilesystem& fs, const Path& p) const;

#if defined(_WIN32)
struct EnvMapEntry
{
std::unordered_map<std::string, std::string> env_map;
Cache<vcpkg::Command, Environment, CommandLess> cmd_cache;
};

Cache<std::vector<std::string>, EnvMapEntry> envs;
#endif

bool m_compiler_tracking;
};
Expand Down
9 changes: 6 additions & 3 deletions locales/messages.json
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,10 @@
"_CMakeToolChainFile.comment": "An example of {path} is /foo/bar.",
"CMakeUsingExportedLibs": "To use exported libraries in CMake projects, add {value} to your CMake command line.",
"_CMakeUsingExportedLibs.comment": "{value} is a CMake command line switch of the form -DFOO=BAR",
"CaptureCmdEnvFailedExitCode": "while trying to capture an environment, '{command_name}' returned {exit_code}.",
"_CaptureCmdEnvFailedExitCode.comment": "An example of {command_name} is install. An example of {exit_code} is 127.",
"CaptureCmdRunFailed": "failed to run '{command_name}' to get environment ({error_msg}).",
"_CaptureCmdRunFailed.comment": "An example of {command_name} is install. An example of {error_msg} is File Not Found.",
"CheckedOutGitSha": "Checked out Git SHA: {commit_sha}",
"_CheckedOutGitSha.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
"CheckedOutObjectMissingManifest": "The checked-out object does not contain a CONTROL file or vcpkg.json file.",
Expand Down Expand Up @@ -1032,6 +1036,8 @@
"InvalidCommitId": "Invalid commit id: {commit_sha}",
"_InvalidCommitId.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949.",
"InvalidDefaultFeatureName": "'default' is a reserved feature name",
"InvalidEnvSetupScripts": "Variable VCPKG_ENVIRONMENT_SETUP_SCRIPTS contains invalid file path: '{path}'. The value must be an absolute path to an existent file.",
"_InvalidEnvSetupScripts.comment": "An example of {path} is /foo/bar.",
"InvalidFeature": "features must be lowercase alphanumeric+hyphens, and not one of the reserved names",
"InvalidFileType": "failed: {path} cannot handle file type",
"_InvalidFileType.comment": "An example of {path} is /foo/bar.",
Expand Down Expand Up @@ -1568,9 +1574,6 @@
"VcpkgSendMetricsButDisabled": "passed --sendmetrics, but metrics are disabled.",
"VcpkgUsage": "usage: vcpkg <command> [--switches] [--options=values] [arguments] @response_file",
"_VcpkgUsage.comment": "This is describing a command line, everything should be localized except 'vcpkg'; symbols like <>s, []s, or --s should be preserved. @response_file should be localized to be consistent with the message named 'ResponseFileCode'.",
"VcvarsRunFailed": "failed to run vcvarsall.bat to get a Visual Studio environment",
"VcvarsRunFailedExitCode": "while trying to get a Visual Studio environment, vcvarsall.bat returned {exit_code}",
"_VcvarsRunFailedExitCode.comment": "An example of {exit_code} is 127.",
"VersionBaselineMismatch": "The latest version is {expected}, but the baseline file contains {actual}.\nRun:\nvcpkg x-add-version {package_name}\ngit add versions\ngit commit -m \"Update version database\"\nto update the baseline version.",
"_VersionBaselineMismatch.comment": "{expected} and {actual} are versions An example of {package_name} is zlib.",
"VersionBuiltinPortTreeEntryMissing": "no version database entry for {package_name} at {expected}; using the checked out ports tree version ({actual}).",
Expand Down
28 changes: 20 additions & 8 deletions src/vcpkg/base/system.process.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1256,38 +1256,47 @@ namespace

namespace vcpkg
{
#if defined(_WIN32)
Environment cmd_execute_and_capture_environment(const Command& cmd, const Environment& env)
{
static StringLiteral magic_string = "cdARN4xjKueKScMy9C6H";

Command actual_cmd = cmd;
#ifdef _WIN32
actual_cmd.raw_arg(Strings::concat(" & echo ", magic_string, " & set"));

#else
actual_cmd.raw_arg(Strings::concat(" && echo ", magic_string, " && printenv"));
#endif
Debug::print("command line: ", actual_cmd.command_line(), "\n");

RedirectedProcessLaunchSettings settings;
settings.environment = env;
#ifdef _WIN32
settings.create_new_console = CreateNewConsole::Yes;
#endif
auto maybe_rc_output = cmd_execute_and_capture_output(actual_cmd, settings);

Neumann-A marked this conversation as resolved.
Show resolved Hide resolved
Neumann-A marked this conversation as resolved.
Show resolved Hide resolved
if (!maybe_rc_output)
{
Checks::msg_exit_with_error(
VCPKG_LINE_INFO, msg::format(msgVcvarsRunFailed).append_raw("\n").append(maybe_rc_output.error()));
Checks::msg_exit_with_error(VCPKG_LINE_INFO,
msgCaptureCmdRunFailed,
msg::command_name = actual_cmd.command_line(),
msg::error_msg = maybe_rc_output.error());
}

auto& rc_output = maybe_rc_output.value_or_exit(VCPKG_LINE_INFO);
Debug::print(rc_output.output, "\n");
if (rc_output.exit_code != 0)
{
Checks::msg_exit_with_error(
VCPKG_LINE_INFO, msgVcvarsRunFailedExitCode, msg::exit_code = rc_output.exit_code);
Checks::msg_exit_with_error(VCPKG_LINE_INFO,
msgCaptureCmdEnvFailedExitCode,
msg::command_name = actual_cmd.command_line(),
msg::exit_code = rc_output.exit_code);
}

auto it = Strings::search(rc_output.output, magic_string);
const char* const last = rc_output.output.data() + rc_output.output.size();

Checks::check_exit(VCPKG_LINE_INFO, it != last);
Checks::check_exit(VCPKG_LINE_INFO, it != last); // magic string not found !
// find the first non-whitespace character after the magic string
it = std::find_if_not(it + magic_string.size(), last, ::isspace);
Checks::check_exit(VCPKG_LINE_INFO, it != last);
Expand All @@ -1299,7 +1308,11 @@ namespace vcpkg
auto equal_it = std::find(it, last, '=');
if (equal_it == last) break;
StringView variable_name(it, equal_it);
#ifdef _WIN32
auto newline_it = std::find(equal_it + 1, last, '\r');
#else
auto newline_it = std::find(equal_it + 1, last, '\n');
#endif
if (newline_it == last) break;
StringView value(equal_it + 1, newline_it);

Expand All @@ -1311,7 +1324,6 @@ namespace vcpkg

return new_env;
}
#endif
} // namespace vcpkg

namespace
Expand Down
1 change: 1 addition & 0 deletions src/vcpkg/cmakevars.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ VCPKG_ENV_PASSTHROUGH=${VCPKG_ENV_PASSTHROUGH}
VCPKG_ENV_PASSTHROUGH_UNTRACKED=${VCPKG_ENV_PASSTHROUGH_UNTRACKED}
VCPKG_LOAD_VCVARS_ENV=${VCPKG_LOAD_VCVARS_ENV}
VCPKG_DISABLE_COMPILER_TRACKING=${VCPKG_DISABLE_COMPILER_TRACKING}
VCPKG_ENVIRONMENT_SETUP_SCRIPTS=${VCPKG_ENVIRONMENT_SETUP_SCRIPTS}
VCPKG_HASH_ADDITIONAL_FILES=${VCPKG_HASH_ADDITIONAL_FILES}
VCPKG_XBOX_CONSOLE_TARGET=${VCPKG_XBOX_CONSOLE_TARGET}
Z_VCPKG_GameDKLatest=$ENV{GameDKLatest}
Expand Down
83 changes: 76 additions & 7 deletions src/vcpkg/commands.build.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -470,16 +470,53 @@ namespace vcpkg
return base_env.cmd_cache.get_lazy(build_env_cmd, [&]() {
const Path& powershell_exe_path = paths.get_tool_exe("powershell-core", out_sink);
auto clean_env = get_modified_clean_environment(base_env.env_map, powershell_exe_path.parent_path());
if (build_env_cmd.empty())
return clean_env;
else
return cmd_execute_and_capture_environment(build_env_cmd, clean_env);
auto action_env =
build_env_cmd.empty() ? clean_env : cmd_execute_and_capture_environment(build_env_cmd, clean_env);
for (const auto& env_setup_script : pre_build_info.environment_setup_scripts)
{
const auto env_setup_cmd = make_setup_env_cmd(paths, env_setup_script);
if (vcpkg::Strings::ends_with(env_setup_script, ".cmake"))
{
ProcessLaunchSettings settings;
settings.environment = action_env;
cmd_execute(env_setup_cmd, settings);
Neumann-A marked this conversation as resolved.
Show resolved Hide resolved
}
else
{
action_env = cmd_execute_and_capture_environment(env_setup_cmd, action_env);
}
}
return action_env;
});
}
#else
const Environment& EnvCache::get_action_env(const VcpkgPaths&, const PreBuildInfo&, const Toolset&)
const Environment& EnvCache::get_action_env(const VcpkgPaths& paths,
const PreBuildInfo& pre_build_info,
const Toolset&)
{
return get_clean_environment();
auto action_env = get_clean_environment();
const auto& base_env = envs.get_lazy(pre_build_info.environment_setup_scripts, []() { return EnvMapEntry{}; });

// I think this should be done differently but I don't exactly know how to build the commands beforehand.
// Can I stack base_env.cmd_cache.get_lazy in a loop?

return base_env.cmd_cache.get_lazy(vcpkg::Command{}, [&]() {
for (const auto& env_setup_script : pre_build_info.environment_setup_scripts)
{
const auto env_setup_cmd = make_setup_env_cmd(paths, env_setup_script);
if (vcpkg::Strings::ends_with(env_setup_script, ".cmake"))
{
ProcessLaunchSettings settings;
settings.environment = action_env;
cmd_execute(env_setup_cmd, settings);
}
else
{
action_env = cmd_execute_and_capture_environment(env_setup_cmd, action_env);
}
}
return action_env;
});
}
#endif

Expand Down Expand Up @@ -589,6 +626,33 @@ namespace vcpkg
#endif
}

vcpkg::Command make_setup_env_cmd(const VcpkgPaths& paths, const Path& script)
{
vcpkg::Command env_setup_cmd;
const auto& fs = paths.get_filesystem();

if (script.is_relative() || !fs.is_regular_file(script))
{
// Throw error
Neumann-A marked this conversation as resolved.
Show resolved Hide resolved
Checks::msg_exit_with_message(VCPKG_LINE_INFO, msgInvalidEnvSetupScripts, msg::path = script);
}

if (script.extension() == ".cmake")
{
env_setup_cmd = vcpkg::make_cmake_cmd(paths, script, {});
}
else
{
#ifdef _WIN32
env_setup_cmd = vcpkg::Command{"cmd"}.string_arg("/d").string_arg("/c");
env_setup_cmd.raw_arg(fmt::format(R"("{}" 2>&1 <NUL)", script));
#else
env_setup_cmd = vcpkg::Command{"."}.raw_arg(fmt::format(R"({} 2>&1 </dev/null)", script));
#endif
}
return env_setup_cmd;
}

static std::vector<PackageSpec> fspecs_to_pspecs(View<FeatureSpec> fspecs)
{
std::set<PackageSpec> set;
Expand Down Expand Up @@ -837,7 +901,7 @@ namespace vcpkg

bool PreBuildInfo::using_vcvars() const
{
return (!external_toolchain_file.has_value() || load_vcvars_env) &&
return (!external_toolchain_file.has_value() || !environment_setup_scripts.empty() || load_vcvars_env) &&
(cmake_system_name.empty() || cmake_system_name == "WindowsStore");
}

Expand Down Expand Up @@ -1833,6 +1897,11 @@ namespace vcpkg
Util::Vectors::append(&passthrough_env_vars, Strings::split(*value, ';'));
}

if (auto value = Util::value_if_set_and_nonempty(cmakevars, CMakeVariableEnvSetupScripts))
{
Util::Vectors::append(&environment_setup_scripts, Strings::split(*value, ';'));
}

Util::assign_if_set_and_nonempty(public_abi_override, cmakevars, CMakeVariablePublicAbiOverride);
if (auto value = Util::value_if_set_and_nonempty(cmakevars, CMakeVariableHashAdditionalFiles))
{
Expand Down
7 changes: 5 additions & 2 deletions src/vcpkg/vcpkgcmdarguments.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -710,8 +710,11 @@ namespace vcpkg
obj.insert(JsonIdVcpkgDisableMetrics, Json::Value::boolean(true));
}

set_environment_variable(EnvironmentVariableXVcpkgRecursiveData,
Json::stringify(obj, Json::JsonStyle::with_spaces(0)));
// Remove newlines from JSON so that environment can be captured on !windows
auto json_str = Json::stringify(obj, Json::JsonStyle::with_spaces(0));
std::replace(json_str.begin(), json_str.end(), '\n', ' ');

set_environment_variable(EnvironmentVariableXVcpkgRecursiveData, std::move(json_str));
}
}

Expand Down
Loading