From 8c1761a7e7c34d18fd0fd8f5509100bfe88bbeb2 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 26 Apr 2023 09:08:30 -0700 Subject: [PATCH 1/9] Refactor Binary Caching --- include/vcpkg-test/util.h | 5 +- include/vcpkg/archives.h | 31 +- include/vcpkg/base/files.h | 24 +- include/vcpkg/base/fwd/files.h | 1 + include/vcpkg/base/message-data.inc.h | 5 +- include/vcpkg/binarycaching.h | 158 +- include/vcpkg/binarycaching.private.h | 39 +- include/vcpkg/commands.build.h | 2 - include/vcpkg/commands.install.h | 1 - include/vcpkg/commands.set-installed.h | 1 - include/vcpkg/dependencies.h | 19 +- include/vcpkg/fwd/binarycaching.h | 13 +- include/vcpkg/vcpkgcmdarguments.h | 22 +- locales/messages.json | 4 +- src/vcpkg-test/binarycaching.cpp | 101 +- src/vcpkg-test/configparser.cpp | 174 +- src/vcpkg-test/dependencies.cpp | 6 +- src/vcpkg-test/plan.cpp | 53 +- src/vcpkg-test/util.cpp | 2 +- src/vcpkg-test/versionplan.cpp | 10 +- src/vcpkg/archives.cpp | 26 +- src/vcpkg/base/downloads.cpp | 1 + src/vcpkg/base/files.cpp | 8 +- src/vcpkg/binarycaching.cpp | 2672 +++++++++++------------- src/vcpkg/commands.build-external.cpp | 4 +- src/vcpkg/commands.build.cpp | 17 +- src/vcpkg/commands.check-support.cpp | 3 +- src/vcpkg/commands.ci.cpp | 7 +- src/vcpkg/commands.dependinfo.cpp | 2 +- src/vcpkg/commands.install.cpp | 32 +- src/vcpkg/commands.set-installed.cpp | 14 +- src/vcpkg/commands.upgrade.cpp | 14 +- src/vcpkg/dependencies.cpp | 23 +- src/vcpkg/export.prefab.cpp | 5 +- src/vcpkg/vcpkgcmdarguments.cpp | 29 +- 35 files changed, 1678 insertions(+), 1850 deletions(-) diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index d38fb11720..4ae4a695f0 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -4,11 +4,14 @@ #include +#include + +#include + #include #include #include #include -#include #include #include diff --git a/include/vcpkg/archives.h b/include/vcpkg/archives.h index be10f3c354..a4aa5e7fb6 100644 --- a/include/vcpkg/archives.h +++ b/include/vcpkg/archives.h @@ -1,12 +1,15 @@ #pragma once +#include #include +#include #include #include +#include #include -#include +#include namespace vcpkg { @@ -27,14 +30,26 @@ namespace vcpkg Filesystem& fs, const ToolCache& tools, MessageSink& status_sink, const Path& archive, const Path& to_path); #endif - // Compress the source directory into the destination file. - ExpectedL compress_directory_to_zip( - Filesystem& fs, const ToolCache& tools, MessageSink& status_sink, const Path& source, const Path& destination); + struct ZipTool + { +#if defined _WIN32 + ZipTool(RemoveFilesystem& fs, const ToolCache& tools, MessageSink& status_sink); +#else + ZipTool(RemoveFilesystem& fs); +#endif + + private: + RemoveFilesystem* fs; +#if defined _WIN32 + Path seven_zip; +#endif + + public: + // Compress the source directory into the destination file. + ExpectedL compress_directory_to_zip(const Path& source, const Path& destination) const; - Command decompress_zip_archive_cmd(const ToolCache& tools, - MessageSink& status_sink, - const Path& dst, - const Path& archive_path); + Command decompress_zip_archive_cmd(const Path& dst, const Path& archive_path) const; + }; std::vector> decompress_in_parallel(View jobs); } diff --git a/include/vcpkg/base/files.h b/include/vcpkg/base/files.h index 065cb1ca68..c189ecb1c8 100644 --- a/include/vcpkg/base/files.h +++ b/include/vcpkg/base/files.h @@ -106,10 +106,23 @@ namespace vcpkg virtual ExpectedL> read_lines(const Path& file_path) const = 0; protected: - ~ILineReader(); + ~ILineReader() = default; }; - struct Filesystem : ILineReader + struct RemoveFilesystem + { + virtual bool remove(const Path& target, std::error_code& ec) = 0; + bool remove(const Path& target, LineInfo li); + + virtual void remove_all(const Path& base, std::error_code& ec, Path& failure_point) = 0; + void remove_all(const Path& base, std::error_code& ec); + void remove_all(const Path& base, LineInfo li); + + protected: + ~RemoveFilesystem() = default; + }; + + struct Filesystem : ILineReader, RemoveFilesystem { virtual std::string read_contents(const Path& file_path, std::error_code& ec) const = 0; std::string read_contents(const Path& file_path, LineInfo li) const; @@ -164,13 +177,6 @@ namespace vcpkg StringLiteral temp_suffix, std::error_code& ec) = 0; - virtual bool remove(const Path& target, std::error_code& ec) = 0; - bool remove(const Path& target, LineInfo li); - - virtual void remove_all(const Path& base, std::error_code& ec, Path& failure_point) = 0; - void remove_all(const Path& base, std::error_code& ec); - void remove_all(const Path& base, LineInfo li); - void remove_all_inside(const Path& base, std::error_code& ec, Path& failure_point); void remove_all_inside(const Path& base, std::error_code& ec); void remove_all_inside(const Path& base, LineInfo li); diff --git a/include/vcpkg/base/fwd/files.h b/include/vcpkg/base/fwd/files.h index dd2be72618..0c8d1e04dd 100644 --- a/include/vcpkg/base/fwd/files.h +++ b/include/vcpkg/base/fwd/files.h @@ -48,6 +48,7 @@ namespace vcpkg struct IExclusiveFileLock; struct ILineReader; struct FileContents; + struct RemoveFilesystem; struct Filesystem; struct NotExtensionCaseSensitive; struct NotExtensionCaseInsensitive; diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 0cb7f2fad6..69e61028c1 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -2282,7 +2282,10 @@ DECLARE_MESSAGE(SpecifyTargetArch, "Specify the target architecture triplet. See 'vcpkg help triplet'.\n(default: '{env_var}')") DECLARE_MESSAGE(StartCodeUnitInContinue, (), "", "found start code unit in continue position") DECLARE_MESSAGE(StoredBinaryCache, (msg::path), "", "Stored binary cache: \"{path}\"") -DECLARE_MESSAGE(StoredBinariesToDestinations, (msg::count), "", "Stored binaries in {count} destinations.") +DECLARE_MESSAGE(StoredBinariesToDestinations, + (msg::count, msg::elapsed), + "", + "Stored binaries in {count} destinations in {elapsed}.") DECLARE_MESSAGE(StoreOptionMissingSha, (), "", "--store option is invalid without a sha512") DECLARE_MESSAGE(SuccessfulyExported, (msg::package_name, msg::path), "", "Exported {package_name} to {path}") DECLARE_MESSAGE(SuggestGitPull, (), "", "The result may be outdated. Run `git pull` to get the latest results.") diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index 6c95010a5c..6583e672d0 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -4,12 +4,14 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -22,15 +24,15 @@ namespace vcpkg { struct CacheStatus { - bool should_attempt_precheck(const IBinaryProvider* sender) const noexcept; - bool should_attempt_restore(const IBinaryProvider* sender) const noexcept; + bool should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept; + bool should_attempt_restore(const IReadBinaryProvider* sender) const noexcept; - bool is_unavailable(size_t total_providers) const noexcept; - const IBinaryProvider* get_available_provider() const noexcept; + bool is_unavailable(const IReadBinaryProvider* sender) const noexcept; + const IReadBinaryProvider* get_available_provider() const noexcept; bool is_restored() const noexcept; - void mark_unavailable(const IBinaryProvider* sender); - void mark_available(const IBinaryProvider* sender) noexcept; + void mark_unavailable(const IReadBinaryProvider* sender); + void mark_available(const IReadBinaryProvider* sender) noexcept; void mark_restored() noexcept; private: @@ -38,67 +40,87 @@ namespace vcpkg // The set of providers who know they do not have the associated cache entry. // Flat vector set because N is tiny. - std::vector m_known_unavailable_providers; // meaningful iff m_status == unknown + std::vector m_known_unavailable_providers; // The provider who affirmatively has the associated cache entry. - const IBinaryProvider* m_available_provider = nullptr; // meaningful iff m_status == available + const IReadBinaryProvider* m_available_provider = nullptr; // meaningful iff m_status == available }; - struct BinaryPackageInformation + struct BinaryPackageReadInfo { - explicit BinaryPackageInformation(const InstallPlanAction& action, Optional nuspec); + explicit BinaryPackageReadInfo(const InstallPlanAction& action); std::string package_abi; PackageSpec spec; std::string raw_version; - // only filled if BinaryCache has a provider that returns true for needs_nuspec_data() - Optional nuspec; + Path package_dir; }; - struct BinaryProviderPushRequest + struct BinaryPackageWriteInfo : BinaryPackageReadInfo { - BinaryPackageInformation info; - Path package_dir; + using BinaryPackageReadInfo::BinaryPackageReadInfo; + + // Filled if BinaryCache has a provider that returns true for needs_nuspec_data() + Optional nuspec; + // Filled if BinaryCache has a provider that returns true for needs_zip_file() + // Note: this can be empty if an error occurred while compressing. + Optional zip_path; }; - struct IBinaryProvider + struct IWriteBinaryProvider { - virtual ~IBinaryProvider() = default; - - /// Attempts to restore the package referenced by `action` into the packages directory. - /// Prerequisite: action has a package_abi() - virtual RestoreResult try_restore(const InstallPlanAction& action) const = 0; + virtual ~IWriteBinaryProvider() = default; /// Called upon a successful build of `action` to store those contents in the binary cache. - /// Prerequisite: action has a package_abi() /// returns the number of successful uploads - virtual size_t push_success(const BinaryProviderPushRequest& request, MessageSink& msg_sink) = 0; + virtual size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) = 0; - /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for - /// executing `actions`. - /// `cache_status` is a vector with the same number of entries of actions, where each index corresponds - /// to the action at the same index in `actions`. The provider must mark the cache status as appropriate. - /// Note: `actions` *might not* have package ABIs (for example, a --head package)! - /// Prerequisite: if `actions[i]` has no package ABI, `cache_status[i]` is nullptr. - virtual void prefetch(View actions, View cache_status) const = 0; + virtual bool needs_nuspec_data() const = 0; + virtual bool needs_zip_file() const = 0; + }; - /// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine - /// missing packages. - /// `cache_status` is a view with the same number of entries of actions, where each index corresponds - /// to the action at the same index in `actions`. The provider must mark the cache status as appropriate. - /// Prerequisite: `actions` have package ABIs. - virtual void precheck(View actions, View cache_status) const = 0; + struct IReadBinaryProvider + { + virtual ~IReadBinaryProvider() = default; - virtual bool needs_nuspec_data() const { return false; } + /// Attempts to restore the package referenced by `action` into the packages directory. + /// + /// Prerequisite: action has a package_abi() + virtual RestoreResult try_restore(const InstallPlanAction& action) const = 0; + + /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for executing + /// `actions`. + /// + /// IBinaryProvider should set out_status[i] to RestoreResult::restored for each fetched package. + /// + /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size() + virtual void prefetch(View actions, Span out_status) const = 0; + + /// Checks whether the `actions` are present in the cache, without restoring them. + /// + /// Used by CI to determine missing packages. For each `i`, out_status[i] should be set to + /// CacheAvailability::available or CacheAvailability::unavailable + /// + /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size() + virtual void precheck(View actions, Span out_status) const = 0; + + /// A user-visible vendor string identifying this provider + virtual StringView vendor() const = 0; }; struct UrlTemplate { std::string url_template; - std::vector headers_for_put; - std::vector headers_for_get; + std::vector headers; LocalizedString valid() const; - std::string instantiate_variables(const BinaryPackageInformation& info) const; + std::string instantiate_variables(const BinaryPackageReadInfo& info) const; + }; + + struct NuGetRepoInfo + { + std::string repo; + std::string branch; + std::string commit; }; struct BinaryConfigParserState @@ -135,28 +157,34 @@ namespace vcpkg std::vector secrets; + // These are filled in after construction by reading from args and environment + std::string nuget_prefix; + bool use_nuget_cache = false; + NuGetRepoInfo nuget_repo_info; + void clear(); }; - ExpectedL create_binary_providers_from_configs_pure(const std::string& env_string, - View args); - ExpectedL>> create_binary_providers_from_configs( - const VcpkgPaths& paths, View args); + ExpectedL parse_binary_provider_configs(const std::string& env_string, + View args); - struct BinaryCache + struct BinaryProviders { - BinaryCache(Filesystem& filesystem); - explicit BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths); + std::vector> read; + std::vector> write; + std::string nuget_prefix; + NuGetRepoInfo nuget_repo; + }; - void install_providers(std::vector>&& providers); - void install_providers_for(const VcpkgCmdArguments& args, const VcpkgPaths& paths); + struct ReadOnlyBinaryCache + { + ReadOnlyBinaryCache() = default; + ReadOnlyBinaryCache(BinaryProviders&& providers); + ReadOnlyBinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info); /// Attempts to restore the package referenced by `action` into the packages directory. RestoreResult try_restore(const InstallPlanAction& action); - /// Called upon a successful build of `action` to store those contents in the binary cache. - void push_success(const InstallPlanAction& action, Path package_dir); - /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for /// executing `actions`. void prefetch(View actions); @@ -166,16 +194,32 @@ namespace vcpkg /// Returns a vector where each index corresponds to the matching index in `actions`. std::vector precheck(View actions); - private: + protected: + BinaryProviders m_config; + std::unordered_map m_status; - std::vector> m_providers; - bool needs_nuspec_data = false; - Filesystem& filesystem; + }; + + struct BinaryCache : ReadOnlyBinaryCache + { + BinaryCache(RemoveFilesystem& fs); + BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info); + BinaryCache(BinaryProviders&& providers, RemoveFilesystem& fs, const ToolCache& tools); + ~BinaryCache(); + + /// Called upon a successful build of `action` to store those contents in the binary cache. + void push_success(const InstallPlanAction& action); + + private: + RemoveFilesystem& m_fs; + Optional m_zip_tool; + bool m_needs_nuspec_data = false; + bool m_needs_zip_file = false; }; ExpectedL parse_download_configuration(const Optional& arg); - std::string generate_nuget_packages_config(const ActionPlan& action); + std::string generate_nuget_packages_config(const ActionPlan& action, StringView prefix); LocalizedString format_help_topic_asset_caching(); LocalizedString format_help_topic_binary_caching(); diff --git a/include/vcpkg/binarycaching.private.h b/include/vcpkg/binarycaching.private.h index 45234a45d9..a11285df2e 100644 --- a/include/vcpkg/binarycaching.private.h +++ b/include/vcpkg/binarycaching.private.h @@ -2,13 +2,13 @@ #include +#include #include #include #include #include -#include namespace vcpkg { @@ -31,40 +31,11 @@ namespace vcpkg std::string nupkg_filename() const { return Strings::concat(id, '.', version, ".nupkg"); } }; - inline NugetReference make_nugetref(const PackageSpec& spec, - StringView raw_version, - StringView abi_tag, - const std::string& prefix) - { - return {Strings::concat(prefix, spec.dir()), format_version_for_nugetref(raw_version, abi_tag)}; - } - inline NugetReference make_nugetref(const BinaryPackageInformation& info, const std::string& prefix) - { - return make_nugetref(info.spec, info.raw_version, info.package_abi, prefix); - } - inline NugetReference make_nugetref(const InstallPlanAction& action, const std::string& prefix) - { - return make_nugetref(action.spec, - action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO) - .source_control_file->core_paragraph->raw_version, - action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi, - prefix); - } - - namespace details - { - struct NuGetRepoInfo - { - std::string repo; - std::string branch; - std::string commit; - }; - - NuGetRepoInfo get_nuget_repo_info_from_env(); - } + NugetReference make_nugetref(const InstallPlanAction& action, StringView prefix); + NuGetRepoInfo get_nuget_repo_info_from_env(); std::string generate_nuspec(const Path& package_dir, const InstallPlanAction& action, - const NugetReference& ref, - details::NuGetRepoInfo rinfo = details::get_nuget_repo_info_from_env()); + StringView id_prefix, + NuGetRepoInfo repo_info); } diff --git a/include/vcpkg/commands.build.h b/include/vcpkg/commands.build.h index 3e3ec6b27d..ff17f1f08a 100644 --- a/include/vcpkg/commands.build.h +++ b/include/vcpkg/commands.build.h @@ -43,14 +43,12 @@ namespace vcpkg const FullPackageSpec& full_spec, Triplet host_triplet, const PathsPortFileProvider& provider, - BinaryCache& binary_cache, const IBuildLogsRecorder& build_logs_recorder, const VcpkgPaths& paths); void perform_and_exit_ex(const VcpkgCmdArguments& args, const FullPackageSpec& full_spec, Triplet host_triplet, const PathsPortFileProvider& provider, - BinaryCache& binary_cache, const IBuildLogsRecorder& build_logs_recorder, const VcpkgPaths& paths); diff --git a/include/vcpkg/commands.install.h b/include/vcpkg/commands.install.h index 90a3932cbd..164c43cef4 100644 --- a/include/vcpkg/commands.install.h +++ b/include/vcpkg/commands.install.h @@ -97,7 +97,6 @@ namespace vcpkg const KeepGoing keep_going, const VcpkgPaths& paths, StatusParagraphs& status_db, - BinaryCache& binary_cache, const IBuildLogsRecorder& build_logs_recorder, const CMakeVars::CMakeVarProvider& var_provider); diff --git a/include/vcpkg/commands.set-installed.h b/include/vcpkg/commands.set-installed.h index 9415c78fc8..32a1212dfb 100644 --- a/include/vcpkg/commands.set-installed.h +++ b/include/vcpkg/commands.set-installed.h @@ -24,7 +24,6 @@ namespace vcpkg::Commands::SetInstalled void perform_and_exit_ex(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const PathsPortFileProvider& provider, - BinaryCache& binary_cache, const CMakeVars::CMakeVarProvider& cmake_vars, ActionPlan action_plan, DryRun dry_run, diff --git a/include/vcpkg/dependencies.h b/include/vcpkg/dependencies.h index 1ce40c73c5..300b5025ac 100644 --- a/include/vcpkg/dependencies.h +++ b/include/vcpkg/dependencies.h @@ -87,7 +87,9 @@ namespace vcpkg std::vector build_failure_messages; Triplet host_triplet; + // only valid with source_control_file_and_location Optional abi_info; + Optional package_dir; }; enum class RemovePlanType @@ -147,14 +149,18 @@ namespace vcpkg struct CreateInstallPlanOptions { - CreateInstallPlanOptions(GraphRandomizer* r, Triplet t) : randomizer(r), host_triplet(t) { } - CreateInstallPlanOptions(Triplet t, UnsupportedPortAction action = UnsupportedPortAction::Error) - : host_triplet(t), unsupported_port_action(action) + CreateInstallPlanOptions(GraphRandomizer* r, Triplet t, const Path& p) + : randomizer(r), host_triplet(t), packages_dir(p) + { + } + CreateInstallPlanOptions(Triplet t, const Path& p, UnsupportedPortAction action = UnsupportedPortAction::Error) + : host_triplet(t), packages_dir(p), unsupported_port_action(action) { } GraphRandomizer* randomizer = nullptr; Triplet host_triplet; + Path packages_dir; UnsupportedPortAction unsupported_port_action = UnsupportedPortAction::Warn; }; @@ -172,13 +178,13 @@ namespace vcpkg const CMakeVars::CMakeVarProvider& var_provider, View specs, const StatusParagraphs& status_db, - const CreateInstallPlanOptions& options = {Triplet{}}); + const CreateInstallPlanOptions& options); ActionPlan create_upgrade_plan(const PortFileProvider& provider, const CMakeVars::CMakeVarProvider& var_provider, const std::vector& specs, const StatusParagraphs& status_db, - const CreateInstallPlanOptions& options = {Triplet{}}); + const CreateInstallPlanOptions& options); ExpectedL create_versioned_install_plan(const IVersionedPortfileProvider& vprovider, const IBaselineProvider& bprovider, @@ -187,8 +193,7 @@ namespace vcpkg const std::vector& deps, const std::vector& overrides, const PackageSpec& toplevel, - Triplet host_triplet, - UnsupportedPortAction unsupported_port_action); + const CreateInstallPlanOptions& options); struct FormattedPlan { diff --git a/include/vcpkg/fwd/binarycaching.h b/include/vcpkg/fwd/binarycaching.h index 70fb40c445..51efde3b91 100644 --- a/include/vcpkg/fwd/binarycaching.h +++ b/include/vcpkg/fwd/binarycaching.h @@ -2,19 +2,20 @@ namespace vcpkg { - enum class RestoreResult + enum class RestoreResult : unsigned char { unavailable, restored, }; - enum class CacheAvailability + enum class CacheAvailability : unsigned char { + unknown, unavailable, available, }; - enum class CacheStatusState + enum class CacheStatusState : unsigned char { unknown, // the cache status of the indicated package ABI is unknown available, // the cache is known to contain the package ABI, but it has not been restored @@ -22,8 +23,10 @@ namespace vcpkg }; struct CacheStatus; - struct IBinaryProvider; + struct IReadBinaryProvider; + struct IWriteBinaryProvider; struct BinaryCache; struct BinaryConfigParserState; - struct BinaryProviderPushRequest; + struct BinaryPackageReadInfo; + struct BinaryPackageWriteInfo; } diff --git a/include/vcpkg/vcpkgcmdarguments.h b/include/vcpkg/vcpkgcmdarguments.h index 46ad754ad0..4ce3c3334e 100644 --- a/include/vcpkg/vcpkgcmdarguments.h +++ b/include/vcpkg/vcpkgcmdarguments.h @@ -133,8 +133,28 @@ namespace vcpkg std::vector cli_overlay_triplets; std::vector env_overlay_triplets; + constexpr static StringLiteral BINARY_SOURCES_ENV = "VCPKG_BINARY_SOURCES"; constexpr static StringLiteral BINARY_SOURCES_ARG = "binarysource"; - std::vector binary_sources; + std::vector cli_binary_sources; + Optional env_binary_sources; + constexpr static StringLiteral ACTIONS_CACHE_URL_ENV = "ACTIONS_CACHE_URL"; + Optional actions_cache_url; + constexpr static StringLiteral ACTIONS_RUNTIME_TOKEN_ENV = "ACTIONS_RUNTIME_TOKEN"; + Optional actions_runtime_token; + constexpr static StringLiteral NUGET_ID_PREFIX_ENV = "X_VCPKG_NUGET_ID_PREFIX"; + Optional nuget_id_prefix; + constexpr static StringLiteral VCPKG_USE_NUGET_CACHE_ENV = "VCPKG_USE_NUGET_CACHE"; + Optional use_nuget_cache; + constexpr static StringLiteral VCPKG_NUGET_REPOSITORY_ENV = "VCPKG_NUGET_REPOSITORY"; + Optional vcpkg_nuget_repository; + constexpr static StringLiteral GITHUB_REPOSITORY_ENV = "GITHUB_REPOSITORY"; + Optional github_repository; + constexpr static StringLiteral GITHUB_SERVER_URL_ENV = "GITHUB_SERVER_URL"; + Optional github_server_url; + constexpr static StringLiteral GITHUB_REF_ENV = "GITHUB_REF"; + Optional github_ref; + constexpr static StringLiteral GITHUB_SHA_ENV = "GITHUB_SHA"; + Optional github_sha; constexpr static StringLiteral CMAKE_SCRIPT_ARG = "cmake-args"; std::vector cmake_args; diff --git a/locales/messages.json b/locales/messages.json index 1d6c25312a..73cc4c6fde 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -1267,8 +1267,8 @@ "_SpecifyTargetArch.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", "StartCodeUnitInContinue": "found start code unit in continue position", "StoreOptionMissingSha": "--store option is invalid without a sha512", - "StoredBinariesToDestinations": "Stored binaries in {count} destinations.", - "_StoredBinariesToDestinations.comment": "An example of {count} is 42.", + "StoredBinariesToDestinations": "Stored binaries in {count} destinations in {elapsed}.", + "_StoredBinariesToDestinations.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", "StoredBinaryCache": "Stored binary cache: \"{path}\"", "_StoredBinaryCache.comment": "An example of {path} is /foo/bar.", "SuccessfulyExported": "Exported {package_name} to {path}", diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 76d6424a81..0b0783492b 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -16,7 +17,7 @@ using namespace vcpkg; -struct KnowNothingBinaryProvider : IBinaryProvider +struct KnowNothingBinaryProvider : IReadBinaryProvider { RestoreResult try_restore(const InstallPlanAction& action) const override { @@ -24,29 +25,25 @@ struct KnowNothingBinaryProvider : IBinaryProvider return RestoreResult::unavailable; } - size_t push_success(const BinaryProviderPushRequest& request, MessageSink&) override + void prefetch(View actions, Span out_status) const override { - CHECK_FALSE(request.info.package_abi.empty()); - return 0; - } - - void prefetch(View actions, View cache_status) const override - { - REQUIRE(actions.size() == cache_status.size()); - for (size_t idx = 0; idx < cache_status.size(); ++idx) + REQUIRE(actions.size() == out_status.size()); + for (size_t idx = 0; idx < out_status.size(); ++idx) { - CHECK(actions[idx].has_package_abi() == (cache_status[idx] != nullptr)); + CHECK(actions[idx]->has_package_abi()); + CHECK(out_status[idx] == RestoreResult::unavailable); } } - void precheck(View actions, View cache_status) const override + void precheck(View actions, Span out_status) const override { - REQUIRE(actions.size() == cache_status.size()); - for (const auto c : cache_status) + REQUIRE(actions.size() == out_status.size()); + for (const auto c : out_status) { - CHECK(c); - c->mark_unavailable(this); + REQUIRE(c == CacheAvailability::unknown); } } + + StringView vendor() const override { return "KnowNothing"; } }; TEST_CASE ("CacheStatus operations", "[BinaryCache]") @@ -57,8 +54,7 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") CacheStatus default_constructed; REQUIRE(default_constructed.should_attempt_precheck(&know_nothing)); REQUIRE(default_constructed.should_attempt_restore(&know_nothing)); - REQUIRE(default_constructed.is_unavailable(0)); - REQUIRE(default_constructed.is_unavailable(1)); + REQUIRE(!default_constructed.is_unavailable(&know_nothing)); REQUIRE(default_constructed.get_available_provider() == nullptr); REQUIRE(!default_constructed.is_restored()); @@ -66,8 +62,7 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") unavailable.mark_unavailable(&know_nothing); REQUIRE(!unavailable.should_attempt_precheck(&know_nothing)); REQUIRE(!unavailable.should_attempt_restore(&know_nothing)); - REQUIRE(!unavailable.is_unavailable(0)); - REQUIRE(unavailable.is_unavailable(1)); + REQUIRE(unavailable.is_unavailable(&know_nothing)); REQUIRE(unavailable.get_available_provider() == nullptr); REQUIRE(!unavailable.is_restored()); @@ -75,8 +70,7 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") available.mark_available(&know_nothing); REQUIRE(!available.should_attempt_precheck(&know_nothing)); REQUIRE(available.should_attempt_restore(&know_nothing)); - REQUIRE(!available.is_unavailable(0)); - REQUIRE(!available.is_unavailable(1)); + REQUIRE(!available.is_unavailable(&know_nothing)); REQUIRE(available.get_available_provider() == &know_nothing); REQUIRE(!available.is_restored()); @@ -84,64 +78,57 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") restored.mark_restored(); REQUIRE(!restored.should_attempt_precheck(&know_nothing)); REQUIRE(!restored.should_attempt_restore(&know_nothing)); - REQUIRE(!restored.is_unavailable(0)); - REQUIRE(!restored.is_unavailable(1)); + REQUIRE(!restored.is_unavailable(&know_nothing)); REQUIRE(restored.get_available_provider() == nullptr); REQUIRE(restored.is_restored()); // CacheStatus(const CacheStatus&); CacheStatus default_copy{default_constructed}; - REQUIRE(default_copy.is_unavailable(0)); + REQUIRE(!default_copy.is_unavailable(&know_nothing)); CacheStatus unavailable_copy{unavailable}; REQUIRE(!unavailable_copy.should_attempt_precheck(&know_nothing)); REQUIRE(!unavailable_copy.should_attempt_restore(&know_nothing)); - REQUIRE(!unavailable_copy.is_unavailable(0)); - REQUIRE(unavailable_copy.is_unavailable(1)); + REQUIRE(unavailable_copy.is_unavailable(&know_nothing)); REQUIRE(unavailable_copy.get_available_provider() == nullptr); REQUIRE(!unavailable_copy.is_restored()); CacheStatus available_copy{available}; REQUIRE(!available_copy.should_attempt_precheck(&know_nothing)); REQUIRE(available_copy.should_attempt_restore(&know_nothing)); - REQUIRE(!available_copy.is_unavailable(0)); - REQUIRE(!available_copy.is_unavailable(1)); + REQUIRE(!available_copy.is_unavailable(&know_nothing)); REQUIRE(available_copy.get_available_provider() == &know_nothing); REQUIRE(!available_copy.is_restored()); CacheStatus restored_copy{restored}; REQUIRE(!restored_copy.should_attempt_precheck(&know_nothing)); REQUIRE(!restored_copy.should_attempt_restore(&know_nothing)); - REQUIRE(!restored_copy.is_unavailable(0)); - REQUIRE(!restored_copy.is_unavailable(1)); + REQUIRE(!restored_copy.is_unavailable(&know_nothing)); REQUIRE(restored_copy.get_available_provider() == nullptr); REQUIRE(restored_copy.is_restored()); // CacheStatus(CacheStatus&&) noexcept; CacheStatus default_move{std::move(default_copy)}; - REQUIRE(default_move.is_unavailable(0)); + REQUIRE(!default_move.is_unavailable(&know_nothing)); CacheStatus unavailable_move{std::move(unavailable_copy)}; REQUIRE(!unavailable_move.should_attempt_precheck(&know_nothing)); REQUIRE(!unavailable_move.should_attempt_restore(&know_nothing)); - REQUIRE(!unavailable_move.is_unavailable(0)); - REQUIRE(unavailable_move.is_unavailable(1)); + REQUIRE(unavailable_move.is_unavailable(&know_nothing)); REQUIRE(unavailable_move.get_available_provider() == nullptr); REQUIRE(!unavailable_move.is_restored()); CacheStatus available_move{std::move(available_copy)}; REQUIRE(!available_move.should_attempt_precheck(&know_nothing)); REQUIRE(available_move.should_attempt_restore(&know_nothing)); - REQUIRE(!available_move.is_unavailable(0)); - REQUIRE(!available_move.is_unavailable(1)); + REQUIRE(!available_move.is_unavailable(&know_nothing)); REQUIRE(available_move.get_available_provider() == &know_nothing); REQUIRE(!available_move.is_restored()); CacheStatus restored_move{std::move(restored_copy)}; REQUIRE(!restored_move.should_attempt_precheck(&know_nothing)); REQUIRE(!restored_move.should_attempt_restore(&know_nothing)); - REQUIRE(!restored_move.is_unavailable(0)); - REQUIRE(!restored_move.is_unavailable(1)); + REQUIRE(!restored_move.is_unavailable(&know_nothing)); REQUIRE(restored_move.get_available_provider() == nullptr); REQUIRE(restored_move.is_restored()); @@ -150,22 +137,19 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") assignee = unavailable; REQUIRE(!assignee.should_attempt_precheck(&know_nothing)); REQUIRE(!assignee.should_attempt_restore(&know_nothing)); - REQUIRE(!assignee.is_unavailable(0)); - REQUIRE(assignee.is_unavailable(1)); + REQUIRE(assignee.is_unavailable(&know_nothing)); REQUIRE(assignee.get_available_provider() == nullptr); REQUIRE(!assignee.is_restored()); assignee = available; REQUIRE(!assignee.should_attempt_precheck(&know_nothing)); REQUIRE(assignee.should_attempt_restore(&know_nothing)); - REQUIRE(!assignee.is_unavailable(0)); - REQUIRE(!assignee.is_unavailable(1)); + REQUIRE(!assignee.is_unavailable(&know_nothing)); REQUIRE(assignee.get_available_provider() == &know_nothing); REQUIRE(!assignee.is_restored()); assignee = restored; REQUIRE(!assignee.should_attempt_precheck(&know_nothing)); REQUIRE(!assignee.should_attempt_restore(&know_nothing)); - REQUIRE(!assignee.is_unavailable(0)); - REQUIRE(!assignee.is_unavailable(1)); + REQUIRE(!assignee.is_unavailable(&know_nothing)); REQUIRE(assignee.get_available_provider() == nullptr); REQUIRE(assignee.is_restored()); @@ -173,22 +157,19 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") assignee = std::move(unavailable); REQUIRE(!assignee.should_attempt_precheck(&know_nothing)); REQUIRE(!assignee.should_attempt_restore(&know_nothing)); - REQUIRE(!assignee.is_unavailable(0)); - REQUIRE(assignee.is_unavailable(1)); + REQUIRE(assignee.is_unavailable(&know_nothing)); REQUIRE(assignee.get_available_provider() == nullptr); REQUIRE(!assignee.is_restored()); assignee = std::move(available); REQUIRE(!assignee.should_attempt_precheck(&know_nothing)); REQUIRE(assignee.should_attempt_restore(&know_nothing)); - REQUIRE(!assignee.is_unavailable(0)); - REQUIRE(!assignee.is_unavailable(1)); + REQUIRE(!assignee.is_unavailable(&know_nothing)); REQUIRE(assignee.get_available_provider() == &know_nothing); REQUIRE(!assignee.is_restored()); assignee = std::move(restored); REQUIRE(!assignee.should_attempt_precheck(&know_nothing)); REQUIRE(!assignee.should_attempt_restore(&know_nothing)); - REQUIRE(!assignee.is_unavailable(0)); - REQUIRE(!assignee.is_unavailable(1)); + REQUIRE(!assignee.is_unavailable(&know_nothing)); REQUIRE(assignee.get_available_provider() == nullptr); REQUIRE(assignee.is_restored()); } @@ -281,7 +262,7 @@ Build-Depends: bzip REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg"); { - auto nuspec = generate_nuspec(pkgPath, ipa, ref, {}); + auto nuspec = generate_nuspec(pkgPath, ipa, "", {}); std::string expected = R"( zlib2_x64-windows @@ -309,7 +290,7 @@ Features: a, b } { - auto nuspec = generate_nuspec(pkgPath, ipa, ref, {"urlvalue"}); + auto nuspec = generate_nuspec(pkgPath, ipa, "", {"urlvalue"}); std::string expected = R"( zlib2_x64-windows @@ -337,7 +318,7 @@ Features: a, b REQUIRE_EQUAL_TEXT(nuspec, expected); } { - auto nuspec = generate_nuspec(pkgPath, ipa, ref, {"urlvalue", "branchvalue", "commitvalue"}); + auto nuspec = generate_nuspec(pkgPath, ipa, "", {"urlvalue", "branchvalue", "commitvalue"}); std::string expected = R"( zlib2_x64-windows @@ -369,10 +350,9 @@ Features: a, b TEST_CASE ("Provider nullptr checks", "[BinaryCache]") { // create a binary cache to test - BinaryCache uut(get_real_filesystem()); - std::vector> providers; - providers.emplace_back(std::make_unique()); - uut.install_providers(std::move(providers)); + BinaryProviders providers; + providers.read.emplace_back(std::make_unique()); + ReadOnlyBinaryCache uut(std::move(providers)); // create an action plan with an action without a package ABI set auto pghs = Paragraphs::parse_paragraphs(R"( @@ -395,7 +375,6 @@ Version: 1.5 InstallPlanAction& ipa_without_abi = install_plan.back(); // test that the binary cache does the right thing. See also CHECKs etc. in KnowNothingBinaryProvider - uut.push_success(ipa_without_abi, {}); // should have no effects CHECK(uut.try_restore(ipa_without_abi) == RestoreResult::unavailable); uut.prefetch(install_plan); // should have no effects } @@ -446,7 +425,7 @@ TEST_CASE ("XmlSerializer", "[XmlSerializer]") TEST_CASE ("generate_nuget_packages_config", "[generate_nuget_packages_config]") { ActionPlan plan; - auto packageconfig = generate_nuget_packages_config(plan); + auto packageconfig = generate_nuget_packages_config(plan, ""); REQUIRE(packageconfig == R"( @@ -471,7 +450,7 @@ Description: a spiffy compression library wrapper plan.install_actions[0].abi_info = AbiInfo{}; plan.install_actions[0].abi_info.get()->package_abi = "packageabi"; - packageconfig = generate_nuget_packages_config(plan); + packageconfig = generate_nuget_packages_config(plan, ""); REQUIRE(packageconfig == R"( @@ -497,7 +476,7 @@ Description: a spiffy compression library wrapper plan.install_actions[1].abi_info = AbiInfo{}; plan.install_actions[1].abi_info.get()->package_abi = "packageabi2"; - packageconfig = generate_nuget_packages_config(plan); + packageconfig = generate_nuget_packages_config(plan, ""); REQUIRE(packageconfig == R"( diff --git a/src/vcpkg-test/configparser.cpp b/src/vcpkg-test/configparser.cpp index f8dc49a72e..9ea3c75da4 100644 --- a/src/vcpkg-test/configparser.cpp +++ b/src/vcpkg-test/configparser.cpp @@ -34,42 +34,42 @@ namespace TEST_CASE ("BinaryConfigParser empty", "[binaryconfigparser]") { - auto parsed = create_binary_providers_from_configs_pure("", {}); + auto parsed = parse_binary_provider_configs("", {}); REQUIRE(parsed.has_value()); } TEST_CASE ("BinaryConfigParser unacceptable provider", "[binaryconfigparser]") { - auto parsed = create_binary_providers_from_configs_pure("unacceptable", {}); + auto parsed = parse_binary_provider_configs("unacceptable", {}); REQUIRE(!parsed.has_value()); } TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("files", {}); + auto parsed = parse_binary_provider_configs("files", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files,relative-path", {}); + auto parsed = parse_binary_provider_configs("files,relative-path", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files,C:foo", {}); + auto parsed = parse_binary_provider_configs("files,C:foo", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH, {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH, {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH ",nonsense", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",nonsense", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH ",read", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); @@ -77,14 +77,14 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") REQUIRE(!Util::Vectors::contains(state.archives_to_write, ABSOLUTE_PATH)); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH ",write", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); REQUIRE(!state.archives_to_write.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH ",readwrite", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); @@ -92,11 +92,11 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH ",readwrite,extra", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",readwrite,extra", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files,,upload", {}); + auto parsed = parse_binary_provider_configs("files,,upload", {}); REQUIRE(!parsed.has_value()); } } @@ -104,36 +104,36 @@ TEST_CASE ("BinaryConfigParser files provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("nuget", {}); + auto parsed = parse_binary_provider_configs("nuget", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nuget,relative-path", {}); + auto parsed = parse_binary_provider_configs("nuget,relative-path", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); validate_readonly_sources(state, "relative-path"); } { - auto parsed = create_binary_providers_from_configs_pure("nuget,http://example.org/", {}); + auto parsed = parse_binary_provider_configs("nuget,http://example.org/", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); validate_readonly_sources(state, "http://example.org/"); } { - auto parsed = create_binary_providers_from_configs_pure("nuget," ABSOLUTE_PATH, {}); + auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH, {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); validate_readonly_sources(state, ABSOLUTE_PATH); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); } { - auto parsed = create_binary_providers_from_configs_pure("nuget," ABSOLUTE_PATH ",nonsense", {}); + auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",nonsense", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nuget," ABSOLUTE_PATH ",readwrite", {}); + auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); CHECK(state.sources_to_read.size() == 1); @@ -145,11 +145,11 @@ TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("nuget," ABSOLUTE_PATH ",readwrite,extra", {}); + auto parsed = parse_binary_provider_configs("nuget," ABSOLUTE_PATH ",readwrite,extra", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nuget,,readwrite", {}); + auto parsed = parse_binary_provider_configs("nuget,,readwrite", {}); REQUIRE(!parsed.has_value()); } } @@ -157,38 +157,38 @@ TEST_CASE ("BinaryConfigParser nuget source provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser nuget timeout", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,3601", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,3601", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"nuget"}}); REQUIRE(state.nugettimeout == std::string{"3601"}); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout", {}); + auto parsed = parse_binary_provider_configs("nugettimeout", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,nonsense", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,nonsense", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,0", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,0", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,12x", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,12x", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,-321", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,-321", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugettimeout,321,123", {}); + auto parsed = parse_binary_provider_configs("nugettimeout,321,123", {}); REQUIRE(!parsed.has_value()); } } @@ -196,27 +196,27 @@ TEST_CASE ("BinaryConfigParser nuget timeout", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig", {}); + auto parsed = parse_binary_provider_configs("nugetconfig", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig,relative-path", {}); + auto parsed = parse_binary_provider_configs("nugetconfig,relative-path", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig,http://example.org/", {}); + auto parsed = parse_binary_provider_configs("nugetconfig,http://example.org/", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig," ABSOLUTE_PATH, {}); + auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH, {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",nonsense", {}); + auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",nonsense", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",read", {}); + auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); CHECK(state.configs_to_write.empty()); @@ -226,7 +226,7 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",write", {}); + auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); CHECK(state.configs_to_read.empty()); @@ -236,7 +236,7 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_write.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",readwrite", {}); + auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); CHECK(state.configs_to_read.size() == 1); @@ -248,11 +248,11 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig," ABSOLUTE_PATH ",readwrite,extra", {}); + auto parsed = parse_binary_provider_configs("nugetconfig," ABSOLUTE_PATH ",readwrite,extra", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("nugetconfig,,readwrite", {}); + auto parsed = parse_binary_provider_configs("nugetconfig,,readwrite", {}); REQUIRE(!parsed.has_value()); } } @@ -260,31 +260,31 @@ TEST_CASE ("BinaryConfigParser nuget config provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("default", {}); + auto parsed = parse_binary_provider_configs("default", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); } { - auto parsed = create_binary_providers_from_configs_pure("default,nonsense", {}); + auto parsed = parse_binary_provider_configs("default,nonsense", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("default,read", {}); + auto parsed = parse_binary_provider_configs("default,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("default,readwrite", {}); + auto parsed = parse_binary_provider_configs("default,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(!state.archives_to_read.empty()); REQUIRE(!state.archives_to_write.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("default,write", {}); + auto parsed = parse_binary_provider_configs("default,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(!state.archives_to_write.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("default,read,extra", {}); + auto parsed = parse_binary_provider_configs("default,read,extra", {}); REQUIRE(!parsed.has_value()); } } @@ -292,11 +292,11 @@ TEST_CASE ("BinaryConfigParser default provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser clear provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("clear", {}); + auto parsed = parse_binary_provider_configs("clear", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("clear,upload", {}); + auto parsed = parse_binary_provider_configs("clear,upload", {}); REQUIRE(!parsed.has_value()); } } @@ -304,11 +304,11 @@ TEST_CASE ("BinaryConfigParser clear provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser interactive provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("interactive", {}); + auto parsed = parse_binary_provider_configs("interactive", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("interactive,read", {}); + auto parsed = parse_binary_provider_configs("interactive,read", {}); REQUIRE(!parsed.has_value()); } } @@ -316,35 +316,35 @@ TEST_CASE ("BinaryConfigParser interactive provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("clear;default", {}); + auto parsed = parse_binary_provider_configs("clear;default", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("clear;default,read", {}); + auto parsed = parse_binary_provider_configs("clear;default,read", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("clear;default,write", {}); + auto parsed = parse_binary_provider_configs("clear;default,write", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("clear;default,readwrite", {}); + auto parsed = parse_binary_provider_configs("clear;default,readwrite", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("clear;default,readwrite;clear;clear", {}); + auto parsed = parse_binary_provider_configs("clear;default,readwrite;clear;clear", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("clear;files,relative;default", {}); + auto parsed = parse_binary_provider_configs("clear;files,relative;default", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure(";;;clear;;;;", {}); + auto parsed = parse_binary_provider_configs(";;;clear;;;;", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure(";;;,;;;;", {}); + auto parsed = parse_binary_provider_configs(";;;,;;;;", {}); REQUIRE(!parsed.has_value()); } } @@ -352,38 +352,38 @@ TEST_CASE ("BinaryConfigParser multiple providers", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser escaping", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure(";;;;;;;`", {}); + auto parsed = parse_binary_provider_configs(";;;;;;;`", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure(";;;;;;;`defaul`t", {}); + auto parsed = parse_binary_provider_configs(";;;;;;;`defaul`t", {}); REQUIRE(parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH "`", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH "`", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH "`,", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH "`,", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH "``", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH "``", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH "```", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH "```", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH "````", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH "````", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"files"}}); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH ",", {}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH ",", {}); REQUIRE(!parsed.has_value()); } } @@ -391,32 +391,28 @@ TEST_CASE ("BinaryConfigParser escaping", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser args", "[binaryconfigparser]") { { - auto parsed = - create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH, std::vector{"clear"}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH, std::vector{"clear"}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{"clear"}); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH, - std::vector{"clear;default"}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH, std::vector{"clear;default"}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"clear"}, {"default"}}); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH, - std::vector{"clear;default,"}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH, std::vector{"clear;default,"}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH, - std::vector{"clear", "clear;default,"}); + auto parsed = + parse_binary_provider_configs("files," ABSOLUTE_PATH, std::vector{"clear", "clear;default,"}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("files," ABSOLUTE_PATH, - std::vector{"clear", "clear"}); + auto parsed = parse_binary_provider_configs("files," ABSOLUTE_PATH, std::vector{"clear", "clear"}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{"clear"}); @@ -427,7 +423,7 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") { UrlTemplate url_temp; { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container,sas", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); @@ -435,23 +431,23 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") REQUIRE(state.secrets == std::vector{"sas"}); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container,?sas", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,?sas", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,,sas", {}); + auto parsed = parse_binary_provider_configs("x-azblob,,sas", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container,sas,invalid", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,invalid", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container,sas,read", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); @@ -460,7 +456,7 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container,sas,write", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); @@ -471,7 +467,7 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_write.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("x-azblob,https://azure/container,sas,readwrite", {}); + auto parsed = parse_binary_provider_configs("x-azblob,https://azure/container,sas,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"azblob"}, {"default"}}); @@ -488,29 +484,29 @@ TEST_CASE ("BinaryConfigParser azblob provider", "[binaryconfigparser]") TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") { { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,gs://my-bucket/", {}); + auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/"}); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); } { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,gs://my-bucket/my-folder", {}); + auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.gcs_read_prefixes == std::vector{"gs://my-bucket/my-folder/"}); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); } { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,", {}); + auto parsed = parse_binary_provider_configs("x-gcs,", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,gs://my-bucket/my-folder,invalid", {}); + auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,invalid", {}); REQUIRE(!parsed.has_value()); } { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,gs://my-bucket/my-folder,read", {}); + auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,read", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); @@ -518,7 +514,7 @@ TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_read.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,gs://my-bucket/my-folder,write", {}); + auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,write", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); @@ -526,7 +522,7 @@ TEST_CASE ("BinaryConfigParser GCS provider", "[binaryconfigparser]") REQUIRE(!state.archives_to_write.empty()); } { - auto parsed = create_binary_providers_from_configs_pure("x-gcs,gs://my-bucket/my-folder,readwrite", {}); + auto parsed = parse_binary_provider_configs("x-gcs,gs://my-bucket/my-folder,readwrite", {}); auto state = parsed.value_or_exit(VCPKG_LINE_INFO); REQUIRE(state.binary_cache_providers == std::set{{"default"}, {"gcs"}}); diff --git a/src/vcpkg-test/dependencies.cpp b/src/vcpkg-test/dependencies.cpp index d5a68a171f..b2dbe43bb6 100644 --- a/src/vcpkg-test/dependencies.cpp +++ b/src/vcpkg-test/dependencies.cpp @@ -229,8 +229,7 @@ static ExpectedL create_versioned_install_plan(const IVersionedPortf deps, overrides, toplevel, - Test::ARM_UWP, - UnsupportedPortAction::Error); + {Test::ARM_UWP, "pkgs", UnsupportedPortAction::Error}); } static ExpectedL create_versioned_install_plan(const IVersionedPortfileProvider& provider, @@ -248,8 +247,7 @@ static ExpectedL create_versioned_install_plan(const IVersionedPortf deps, overrides, toplevel, - Test::ARM_UWP, - UnsupportedPortAction::Error); + {Test::ARM_UWP, "pkgs", UnsupportedPortAction::Error}); } TEST_CASE ("basic version install single", "[versionplan]") diff --git a/src/vcpkg-test/plan.cpp b/src/vcpkg-test/plan.cpp index 713b61bbd1..f67eb35ac3 100644 --- a/src/vcpkg-test/plan.cpp +++ b/src/vcpkg-test/plan.cpp @@ -58,6 +58,34 @@ static void remove_plan_check(RemovePlanAction& plan, std::string pkg_name, Trip REQUIRE(pkg_name == plan.spec.name()); } +static ActionPlan create_feature_install_plan(const PortFileProvider& port_provider, + const CMakeVars::CMakeVarProvider& var_provider, + View specs, + const StatusParagraphs& status_db) +{ + const CreateInstallPlanOptions create_options{Test::X64_ANDROID, "pkg"}; + return create_feature_install_plan(port_provider, var_provider, specs, status_db, create_options); +} + +static ActionPlan create_feature_install_plan(const PortFileProvider& port_provider, + const CMakeVars::CMakeVarProvider& var_provider, + View specs, + const StatusParagraphs& status_db, + Triplet host_triplet) +{ + const CreateInstallPlanOptions create_options{host_triplet, "pkg"}; + return create_feature_install_plan(port_provider, var_provider, specs, status_db, create_options); +} + +static ActionPlan create_upgrade_plan(const PortFileProvider& provider, + const CMakeVars::CMakeVarProvider& var_provider, + const std::vector& specs, + const StatusParagraphs& status_db) +{ + const CreateInstallPlanOptions create_options{Test::X64_ANDROID, "pkg"}; + return create_upgrade_plan(provider, var_provider, specs, status_db, create_options); +} + TEST_CASE ("basic install scheme", "[plan]") { std::vector> status_paragraphs; @@ -1022,16 +1050,16 @@ TEST_CASE ("self-referencing scheme", "[plan]") SECTION ("basic") { - auto install_plan = create_feature_install_plan( - map_port, var_provider, Test::parse_test_fspecs("a"), {}, {{}, Test::X64_WINDOWS}); + auto install_plan = + create_feature_install_plan(map_port, var_provider, Test::parse_test_fspecs("a"), {}, Test::X64_WINDOWS); REQUIRE(install_plan.size() == 1); REQUIRE(install_plan.install_actions.at(0).spec == spec_a); } SECTION ("qualified") { - auto install_plan = create_feature_install_plan( - map_port, var_provider, Test::parse_test_fspecs("b"), {}, {{}, Test::X64_WINDOWS}); + auto install_plan = + create_feature_install_plan(map_port, var_provider, Test::parse_test_fspecs("b"), {}, Test::X64_WINDOWS); REQUIRE(install_plan.size() == 1); REQUIRE(install_plan.install_actions.at(0).spec == spec_b); @@ -1056,7 +1084,7 @@ TEST_CASE ("basic tool port scheme", "[plan]") var_provider, Test::parse_test_fspecs("a"), StatusParagraphs(std::move(status_paragraphs)), - {{}, Test::X64_WINDOWS}); + Test::X64_WINDOWS); REQUIRE(install_plan.size() == 3); REQUIRE(install_plan.install_actions.at(0).spec.name() == "c"); @@ -1085,8 +1113,7 @@ TEST_CASE ("basic existing tool port scheme", "[plan]") MapPortFileProvider map_port(spec_map.map); - auto install_plan = - create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, {{}, Test::X64_WINDOWS}); + auto install_plan = create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, Test::X64_WINDOWS); REQUIRE(install_plan.size() == 1); REQUIRE(install_plan.install_actions.at(0).spec == spec_a); @@ -1101,16 +1128,14 @@ TEST_CASE ("basic existing tool port scheme", "[plan]") MapPortFileProvider map_port(spec_map.map); - auto install_plan = - create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, {{}, Test::X64_WINDOWS}); + auto install_plan = create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, Test::X64_WINDOWS); REQUIRE(install_plan.size() == 2); REQUIRE(install_plan.install_actions.at(0).spec.name() == "a"); REQUIRE(install_plan.install_actions.at(0).spec.triplet() == Test::X64_WINDOWS); REQUIRE(install_plan.install_actions.at(1).spec == spec_a); - install_plan = - create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, {{}, Test::X86_WINDOWS}); + install_plan = create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, Test::X86_WINDOWS); REQUIRE(install_plan.size() == 1); REQUIRE(install_plan.install_actions.at(0).spec == spec_a); @@ -1126,8 +1151,7 @@ TEST_CASE ("basic existing tool port scheme", "[plan]") MapPortFileProvider map_port(spec_map.map); - auto install_plan = - create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, {{}, Test::ARM_UWP}); + auto install_plan = create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, Test::ARM_UWP); REQUIRE(install_plan.size() == 2); REQUIRE(install_plan.install_actions.at(0).spec.name() == "b"); @@ -1146,8 +1170,7 @@ TEST_CASE ("basic existing tool port scheme", "[plan]") MapPortFileProvider map_port(spec_map.map); - auto install_plan = - create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, {{}, Test::X64_WINDOWS}); + auto install_plan = create_feature_install_plan(map_port, var_provider, fspecs_a, status_db, Test::X64_WINDOWS); REQUIRE(install_plan.size() == 1); REQUIRE(install_plan.install_actions.at(0).spec == spec_a); diff --git a/src/vcpkg-test/util.cpp b/src/vcpkg-test/util.cpp index 9f4cd35e1d..68790e03f2 100644 --- a/src/vcpkg-test/util.cpp +++ b/src/vcpkg-test/util.cpp @@ -5,6 +5,7 @@ #include #include +#include #include @@ -265,5 +266,4 @@ namespace vcpkg::Test std::string path = "$"; check_json_eq(l, r, path, true); } - } diff --git a/src/vcpkg-test/versionplan.cpp b/src/vcpkg-test/versionplan.cpp index 747bcd3829..d7267658a4 100644 --- a/src/vcpkg-test/versionplan.cpp +++ b/src/vcpkg-test/versionplan.cpp @@ -91,11 +91,17 @@ TEST_CASE ("qualified dependency", "[dependencies]") var_provider.dep_info_vars[{"a", Triplet::from_canonical_name("x64-linux")}].emplace("VCPKG_CMAKE_SYSTEM_NAME", "Linux"); - auto plan = vcpkg::create_feature_install_plan(map_port, var_provider, Test::parse_test_fspecs("a"), {}); + const CreateInstallPlanOptions create_options{Test::X64_ANDROID, "pkg"}; + + auto plan = + vcpkg::create_feature_install_plan(map_port, var_provider, Test::parse_test_fspecs("a"), {}, create_options); REQUIRE(plan.install_actions.size() == 2); REQUIRE(plan.install_actions.at(0).feature_list == std::vector{"core"}); + REQUIRE(plan.install_actions[0].package_dir == "pkg/b_x86-windows"); - auto plan2 = vcpkg::create_feature_install_plan(map_port, var_provider, Test::parse_test_fspecs("a:x64-linux"), {}); + auto plan2 = vcpkg::create_feature_install_plan( + map_port, var_provider, Test::parse_test_fspecs("a:x64-linux"), {}, create_options); REQUIRE(plan2.install_actions.size() == 2); REQUIRE(plan2.install_actions.at(0).feature_list == std::vector{"b1", "core"}); + REQUIRE(plan.install_actions[0].package_dir == "pkg/b_x64-linux"); } diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index b6f4d18594..d7c6326e69 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -293,21 +293,16 @@ namespace vcpkg fs.rename_with_retry(to_path_partial, to_path, VCPKG_LINE_INFO); } - ExpectedL compress_directory_to_zip( - Filesystem& fs, const ToolCache& tools, MessageSink& status_sink, const Path& source, const Path& destination) + ExpectedL ZipTool::compress_directory_to_zip(const Path& source, const Path& destination) const { - fs.remove(destination, VCPKG_LINE_INFO); + fs->remove(destination, VCPKG_LINE_INFO); #if defined(_WIN32) - auto&& seven_zip_exe = tools.get_tool_path(Tools::SEVEN_ZIP, status_sink); - return flatten(cmd_execute_and_capture_output( - Command{seven_zip_exe}.string_arg("a").string_arg(destination).string_arg(source / "*"), + Command{seven_zip}.string_arg("a").string_arg(destination).string_arg(source / "*"), default_working_directory, get_clean_environment()), Tools::SEVEN_ZIP); #else - (void)tools; - (void)status_sink; return flatten(cmd_execute_and_capture_output(Command{"zip"} .string_arg("--quiet") .string_arg("-y") @@ -321,22 +316,21 @@ namespace vcpkg #endif } - Command decompress_zip_archive_cmd(const ToolCache& tools, - MessageSink& status_sink, - const Path& dst, - const Path& archive_path) + ZipTool::ZipTool(RemoveFilesystem& fs, const ToolCache& cache, MessageSink& status_sink) + : fs(&fs), seven_zip(cache.get_tool_path(Tools::SEVEN_ZIP, status_sink)) + { + } + + Command ZipTool::decompress_zip_archive_cmd(const Path& dst, const Path& archive_path) const { Command cmd; #if defined(_WIN32) - auto&& seven_zip_exe = tools.get_tool_path(Tools::SEVEN_ZIP, status_sink); - cmd.string_arg(seven_zip_exe) + cmd.string_arg(seven_zip) .string_arg("x") .string_arg(archive_path) .string_arg("-o" + dst.native()) .string_arg("-y"); #else - (void)tools; - (void)status_sink; cmd.string_arg("unzip").string_arg("-qq").string_arg(archive_path).string_arg("-d" + dst.native()); #endif return cmd; diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 81f75de535..4063f7086b 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -435,6 +435,7 @@ namespace vcpkg url_heads_inner({urls.begin() + i, urls.end()}, headers, &ret, secrets); } + Checks::check_exit(VCPKG_LINE_INFO, ret.size() == urls.size()); return ret; } diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index 5f0618409c..de72950b5d 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -1534,8 +1534,6 @@ namespace vcpkg int WriteFilePointer::put(int c) const noexcept { return ::fputc(c, m_fs); } - ILineReader::~ILineReader() = default; - std::string Filesystem::read_contents(const Path& file_path, LineInfo li) const { std::error_code ec; @@ -1715,7 +1713,7 @@ namespace vcpkg } } - bool Filesystem::remove(const Path& target, LineInfo li) + bool RemoveFilesystem::remove(const Path& target, LineInfo li) { std::error_code ec; auto r = this->remove(target, ec); @@ -1907,7 +1905,7 @@ namespace vcpkg } } - void Filesystem::remove_all(const Path& base, LineInfo li) + void RemoveFilesystem::remove_all(const Path& base, LineInfo li) { std::error_code ec; Path failure_point; @@ -1924,7 +1922,7 @@ namespace vcpkg } } - void Filesystem::remove_all(const Path& base, std::error_code& ec) + void RemoveFilesystem::remove_all(const Path& base, std::error_code& ec) { Path failure_point; this->remove_all(base, ec, failure_point); diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 9123926750..9863b633da 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include +#include using namespace vcpkg; @@ -184,20 +186,16 @@ namespace return ret; } - static const std::string& get_nuget_prefix() + NugetReference make_nugetref(const PackageSpec& spec, StringView raw_version, StringView abi_tag, StringView prefix) { - static std::string nuget_prefix = []() { - auto x = get_environment_variable("X_VCPKG_NUGET_ID_PREFIX").value_or(""); - if (!x.empty()) - { - x.push_back('_'); - } - return x; - }(); - return nuget_prefix; + return {Strings::concat(prefix, spec.dir()), format_version_for_nugetref(raw_version, abi_tag)}; + } + NugetReference make_nugetref(const BinaryPackageReadInfo& info, StringView prefix) + { + return make_nugetref(info.spec, info.raw_version, info.package_abi, prefix); } - static void clean_prepare_dir(Filesystem& fs, const Path& dir) + void clean_prepare_dir(Filesystem& fs, const Path& dir) { fs.remove_all(dir, VCPKG_LINE_INFO); if (!fs.create_directories(dir, VCPKG_LINE_INFO)) @@ -206,89 +204,83 @@ namespace } } - static Path make_temp_archive_path(const Path& buildtrees, const PackageSpec& spec) + Path make_temp_archive_path(const Path& buildtrees, const PackageSpec& spec) { return buildtrees / spec.name() / (spec.triplet().to_string() + ".zip"); } - struct ArchivesBinaryProvider : IBinaryProvider - { - ArchivesBinaryProvider(const VcpkgPaths& paths, - std::vector&& read_dirs, - std::vector&& write_dirs, - std::vector&& put_url_templates, - const std::vector& secrets) - : paths(paths) - , m_read_dirs(std::move(read_dirs)) - , m_write_dirs(std::move(write_dirs)) - , m_put_url_templates(std::move(put_url_templates)) - , m_secrets(secrets) - { - } + Path files_archive_subpath(const std::string& abi) { return Path(abi.substr(0, 2)) / (abi + ".zip"); } - static Path make_archive_subpath(const std::string& abi) { return Path(abi.substr(0, 2)) / (abi + ".zip"); } + struct FilesWriteBinaryProvider : IWriteBinaryProvider + { + FilesWriteBinaryProvider(Filesystem& fs, std::vector&& dirs) : m_fs(fs), m_dirs(std::move(dirs)) { } - void prefetch(View actions, View cache_status) const override + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override { - std::vector to_try_restore_idxs; - std::vector to_try_restore; + if (!request.zip_path) return 0; + const auto& zip_path = request.zip_path.value_or_exit(VCPKG_LINE_INFO); + const auto archive_subpath = files_archive_subpath(request.package_abi); - for (const auto& archives_root_dir : m_read_dirs) + int count_stored = 0; + for (const auto& archives_root_dir : m_dirs) { - const ElapsedTimer timer; - to_try_restore_idxs.clear(); - to_try_restore.clear(); - for (size_t idx = 0; idx < cache_status.size(); ++idx) + const auto archive_path = archives_root_dir / archive_subpath; + std::error_code ec; + m_fs.copy_file(zip_path, archive_path, CopyOptions::overwrite_existing, ec); + if (ec) { - auto idx_cache_status = cache_status[idx]; - if (idx_cache_status && idx_cache_status->should_attempt_restore(this)) - { - to_try_restore_idxs.push_back(idx); - to_try_restore.push_back(&actions[idx]); - } + ec.clear(); + m_fs.create_directories(archive_path.parent_path(), IgnoreErrors{}); + m_fs.copy_file(zip_path, archive_path, CopyOptions::overwrite_existing, ec); } - auto results = try_restore_n(to_try_restore, archives_root_dir); - int num_restored = 0; - for (size_t n = 0; n < to_try_restore.size(); ++n) + if (ec) { - if (results[n] == RestoreResult::restored) - { - cache_status[to_try_restore_idxs[n]]->mark_restored(); - ++num_restored; - } + msg_sink.println(Color::warning, + msg::format(msgFailedToStoreBinaryCache, msg::path = archive_path) + .append_raw('\n') + .append_raw(ec.message())); + } + else + { + count_stored++; } - msg::println(msgRestoredPackagesFromVendor, - msg::count = num_restored, - msg::elapsed = timer.elapsed(), - msg::value = archives_root_dir.native()); } + return count_stored; } - std::vector try_restore_n(View actions, - const Path& archives_root_dir) const + bool needs_nuspec_data() const override { return false; } + bool needs_zip_file() const override { return true; } + + private: + Filesystem& m_fs; + std::vector m_dirs; + }; + + // This middleware class contains logic for BinaryProviders that operate on zip files. + // Derived classes must implement: + // - acquire_zips() + // - post_decompress() + // - IReadBinaryProvider::precheck() + struct ZipReadBinaryProvider : IReadBinaryProvider + { + ZipReadBinaryProvider(ZipTool zip, Filesystem& fs) : m_zip(std::move(zip)), m_fs(fs) { } + + void prefetch(View actions, Span out_status) const override { - auto& fs = paths.get_filesystem(); - std::vector results(actions.size(), RestoreResult::unavailable); - std::vector action_idxs; + const ElapsedTimer timer; + std::vector> zip_paths(actions.size(), nullopt); + acquire_zips(actions, zip_paths); + std::vector jobs; - std::vector archive_paths; + std::vector action_idxs; for (size_t i = 0; i < actions.size(); ++i) { - const auto& action = *actions[i]; - const auto& spec = action.spec; - const auto& abi_tag = action.package_abi().value_or_exit(VCPKG_LINE_INFO); - const auto archive_subpath = make_archive_subpath(abi_tag); - auto archive_path = archives_root_dir / archive_subpath; - if (fs.exists(archive_path, IgnoreErrors{})) - { - auto pkg_path = paths.package_dir(spec); - clean_prepare_dir(fs, pkg_path); - jobs.push_back( - decompress_zip_archive_cmd(paths.get_tool_cache(), stdout_sink, pkg_path, archive_path)); - action_idxs.push_back(i); - archive_paths.push_back(std::move(archive_path)); - } + if (!zip_paths[i]) continue; + const auto& pkg_path = actions[i]->package_dir.value_or_exit(VCPKG_LINE_INFO); + clean_prepare_dir(m_fs, pkg_path); + jobs.push_back(m_zip.decompress_zip_archive_cmd(pkg_path, *zip_paths[i].get())); + action_idxs.push_back(i); } auto job_results = decompress_in_parallel(jobs); @@ -296,27 +288,23 @@ namespace for (size_t j = 0; j < jobs.size(); ++j) { const auto i = action_idxs[j]; - const auto& archive_result = job_results[j]; - if (archive_result) + const auto& zip_path = zip_paths[i].value_or_exit(VCPKG_LINE_INFO); + if (job_results[j]) { - results[i] = RestoreResult::restored; - Debug::print("Restored ", archive_paths[j].native(), '\n'); + Debug::print("Restored ", zip_path, '\n'); + out_status[i] = RestoreResult::restored; } else { - if (actions[i]->build_options.purge_decompress_failure == PurgeDecompressFailure::YES) - { - Debug::print( - "Failed to decompress archive package; purging: ", archive_paths[j].native(), '\n'); - fs.remove(archive_paths[j], IgnoreErrors{}); - } - else - { - Debug::print("Failed to decompress archive package: ", archive_paths[j].native(), '\n'); - } + Debug::print("Failed to decompress archive package: ", zip_path, '\n'); } + post_decompress(*actions[i], zip_path, job_results[j].has_value()); } - return results; + size_t num_restored = std::count(out_status.begin(), out_status.end(), RestoreResult::restored); + msg::println(msgRestoredPackagesFromVendor, + msg::count = num_restored, + msg::elapsed = timer.elapsed(), + msg::value = vendor()); } RestoreResult try_restore(const InstallPlanAction& action) const override @@ -324,281 +312,264 @@ namespace // Note: this method is almost never called -- it will only be called if another provider promised to // restore a package but then failed at runtime auto p_action = &action; - for (const auto& archives_root_dir : m_read_dirs) + Optional zip_path; + acquire_zips({&p_action, 1}, {&zip_path, 1}); + if (auto path = zip_path.get()) { - if (try_restore_n({&p_action, 1}, archives_root_dir)[0] == RestoreResult::restored) + const auto& package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO); + auto cmd = m_zip.decompress_zip_archive_cmd(package_dir, *path); + auto result = decompress_in_parallel({&cmd, 1}); + post_decompress(action, *path, result[0].has_value()); + if (result[0]) { - msg::println(msgRestoredPackage, msg::path = archives_root_dir.native()); + msg::println(msgRestoredPackage, msg::path = vendor()); return RestoreResult::restored; } } return RestoreResult::unavailable; } - size_t push_success(const BinaryProviderPushRequest& request, MessageSink& msg_sink) override - { - if (m_write_dirs.empty() && m_put_url_templates.empty()) - { - return 0; - } + // Download/find the zip files corresponding with a set of actions + virtual void acquire_zips(View actions, Span> out_zip_paths) const = 0; - const auto& abi_tag = request.info.package_abi; - auto& spec = request.info.spec; - auto& fs = paths.get_filesystem(); - const auto archive_subpath = make_archive_subpath(abi_tag); - const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec); - auto compress_result = - compress_directory_to_zip(fs, paths.get_tool_cache(), msg_sink, request.package_dir, tmp_archive_path); - if (!compress_result) - { - msg_sink.println(Color::warning, - msg::format_warning(msgCompressFolderFailed, msg::path = request.package_dir) - .append_raw(' ') - .append_raw(compress_result.error())); - return 0; - } - int count_stored = 0; - for (auto&& put_url_template : m_put_url_templates) - { - auto url = put_url_template.instantiate_variables(request.info); - auto maybe_success = put_file(fs, url, m_secrets, put_url_template.headers_for_put, tmp_archive_path); - if (maybe_success) - { - count_stored++; - continue; - } + // Called for each zip file after `acquire_zips()`, should implement cleanup policies + virtual void post_decompress(const InstallPlanAction& action, + const Path& archive_path, + bool succeeded) const = 0; - msg_sink.println(Color::warning, maybe_success.error()); - } + protected: + ZipTool m_zip; + Filesystem& m_fs; + }; - for (const auto& archives_root_dir : m_write_dirs) - { - const auto archive_path = archives_root_dir / archive_subpath; - fs.create_directories(archive_path.parent_path(), IgnoreErrors{}); - std::error_code ec; - if (m_write_dirs.size() > 1) - { - fs.copy_file(tmp_archive_path, archive_path, CopyOptions::overwrite_existing, ec); - } - else - { - fs.rename_or_copy(tmp_archive_path, archive_path, ".tmp", ec); - } + struct FilesReadBinaryProvider : ZipReadBinaryProvider + { + FilesReadBinaryProvider(ZipTool zip, Filesystem& fs, Path&& dir) + : ZipReadBinaryProvider(std::move(zip), fs), m_dir(std::move(dir)) + { + } - if (ec) - { - msg_sink.println(Color::warning, - msg::format(msgFailedToStoreBinaryCache, msg::path = archive_path) - .append_raw('\n') - .append_raw(ec.message())); - } - else + void acquire_zips(View actions, Span> out_zip_paths) const override + { + for (size_t i = 0; i < actions.size(); ++i) + { + const auto& abi_tag = actions[i]->package_abi().value_or_exit(VCPKG_LINE_INFO); + auto archive_path = m_dir / files_archive_subpath(abi_tag); + if (m_fs.exists(archive_path, IgnoreErrors{})) { - count_stored++; + out_zip_paths[i] = std::move(archive_path); } } - // In the case of 1 write dir, the file will be moved instead of copied - if (m_write_dirs.size() != 1) + } + + void post_decompress(const InstallPlanAction& action, const Path& archive_path, bool succeeded) const override + { + if (!succeeded && action.build_options.purge_decompress_failure == PurgeDecompressFailure::YES) { - fs.remove(tmp_archive_path, IgnoreErrors{}); + Debug::print("Failed to decompress archive package; purging: ", archive_path, '\n'); + m_fs.remove(archive_path, IgnoreErrors{}); } - return count_stored; } - void precheck(View actions, View cache_status) const override + void precheck(View actions, Span cache_status) const override { - auto& fs = paths.get_filesystem(); for (size_t idx = 0; idx < actions.size(); ++idx) { - const auto& action = actions[idx]; + const auto& action = *actions[idx]; const auto& abi_tag = action.package_abi().value_or_exit(VCPKG_LINE_INFO); - if (!cache_status[idx]->should_attempt_precheck(this)) - { - continue; - } - const auto archive_subpath = make_archive_subpath(abi_tag); bool any_available = false; - for (auto&& archives_root_dir : m_read_dirs) + if (m_fs.exists(m_dir / files_archive_subpath(abi_tag), IgnoreErrors{})) { - if (fs.exists(archives_root_dir / archive_subpath, IgnoreErrors{})) - { - any_available = true; - break; - } + any_available = true; } - if (any_available) - { - cache_status[idx]->mark_available(this); - } - else - { - cache_status[idx]->mark_unavailable(this); - } + cache_status[idx] = any_available ? CacheAvailability::available : CacheAvailability::unavailable; } } + StringView vendor() const override { return m_dir; } + private: - const VcpkgPaths& paths; - std::vector m_read_dirs; - std::vector m_write_dirs; - std::vector m_put_url_templates; - std::vector m_secrets; + Path m_dir; }; - struct HttpGetBinaryProvider : IBinaryProvider + + struct HTTPPutBinaryProvider : IWriteBinaryProvider { - HttpGetBinaryProvider(const VcpkgPaths& paths, - std::vector&& url_templates, + HTTPPutBinaryProvider(const Filesystem& fs, + std::vector&& urls, const std::vector& secrets) - : paths(paths), m_url_templates(std::move(url_templates)), m_secrets(secrets) + : m_fs(fs), m_urls(std::move(urls)), m_secrets(secrets) { } - RestoreResult try_restore(const InstallPlanAction&) const override { return RestoreResult::unavailable; } + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override + { + if (!request.zip_path) return 0; + const auto& zip_path = *request.zip_path.get(); + size_t count_stored = 0; + for (auto&& templ : m_urls) + { + auto url = templ.instantiate_variables(request); + auto maybe_success = put_file(m_fs, url, m_secrets, templ.headers, zip_path); + if (maybe_success) + count_stored++; + else + msg_sink.println(Color::warning, maybe_success.error()); + } + return count_stored; + } + + bool needs_nuspec_data() const override { return false; } + bool needs_zip_file() const override { return true; } + + private: + const Filesystem& m_fs; + std::vector m_urls; + std::vector m_secrets; + }; - size_t push_success(const BinaryProviderPushRequest&, MessageSink&) override { return 0; } + struct HttpGetBinaryProvider : ZipReadBinaryProvider + { + HttpGetBinaryProvider(ZipTool zip, + Filesystem& fs, + const Path& buildtrees, + UrlTemplate&& url_template, + const std::vector& secrets) + : ZipReadBinaryProvider(std::move(zip), fs) + , m_buildtrees(buildtrees) + , m_url_template(std::move(url_template)) + , m_secrets(secrets) + { + } - void prefetch(View actions, View cache_status) const override + void acquire_zips(View actions, Span> out_zip_paths) const override { - const ElapsedTimer timer; - auto& fs = paths.get_filesystem(); - size_t this_restore_count = 0; std::vector> url_paths; - std::vector url_indices; - for (auto&& url_template : m_url_templates) + for (size_t idx = 0; idx < actions.size(); ++idx) { - url_paths.clear(); - url_indices.clear(); - for (size_t idx = 0; idx < cache_status.size(); ++idx) - { - auto this_cache_status = cache_status[idx]; - if (!this_cache_status || !this_cache_status->should_attempt_restore(this)) - { - continue; - } - - auto&& action = actions[idx]; - clean_prepare_dir(fs, paths.package_dir(action.spec)); - auto uri = url_template.instantiate_variables(BinaryPackageInformation{action, ""}); - url_paths.emplace_back(std::move(uri), make_temp_archive_path(paths.buildtrees(), action.spec)); - url_indices.push_back(idx); - } - - if (url_paths.empty()) break; + auto&& action = *actions[idx]; + auto uri = m_url_template.instantiate_variables(BinaryPackageReadInfo{action}); + url_paths.emplace_back(std::move(uri), make_temp_archive_path(m_buildtrees, action.spec)); + } - msg::println(msgAttemptingToFetchPackagesFromVendor, - msg::count = url_paths.size(), - msg::vendor = "HTTP servers"); + auto codes = download_files(m_fs, url_paths, m_url_template.headers); - auto codes = download_files(fs, url_paths, url_template.headers_for_get); - std::vector action_idxs; - std::vector jobs; - for (size_t i = 0; i < codes.size(); ++i) - { - if (codes[i] == 200) - { - action_idxs.push_back(i); - jobs.push_back(decompress_zip_archive_cmd(paths.get_tool_cache(), - stdout_sink, - paths.package_dir(actions[url_indices[i]].spec), - url_paths[i].second)); - } - } - auto job_results = decompress_in_parallel(jobs); - for (size_t j = 0; j < jobs.size(); ++j) + for (size_t i = 0; i < codes.size(); ++i) + { + if (codes[i] == 200) { - const auto i = action_idxs[j]; - if (job_results[j]) - { - ++this_restore_count; - fs.remove(url_paths[i].second, VCPKG_LINE_INFO); - cache_status[url_indices[i]]->mark_restored(); - } - else - { - Debug::print("Failed to decompress ", url_paths[i].second, '\n'); - } + out_zip_paths[i] = url_paths[i].second; } } + } - msg::println(msgRestoredPackagesFromVendor, - msg::count = this_restore_count, - msg::elapsed = timer.elapsed(), - msg::value = "HTTP servers"); + void post_decompress(const InstallPlanAction&, const Path& archive_path, bool) const override + { + m_fs.remove(archive_path, VCPKG_LINE_INFO); } - void precheck(View actions, View cache_status) const override + void precheck(View actions, Span out_status) const override { - std::vector actions_present{actions.size()}; std::vector urls; - std::vector url_indices; - for (auto&& url_template : m_url_templates) + for (size_t idx = 0; idx < actions.size(); ++idx) { - urls.clear(); - url_indices.clear(); - for (size_t idx = 0; idx < actions.size(); ++idx) - { - if (!cache_status[idx]->should_attempt_precheck(this)) - { - continue; - } - - urls.push_back(url_template.instantiate_variables(BinaryPackageInformation{actions[idx], {}})); - url_indices.push_back(idx); - } - - if (urls.empty()) - { - return; - } - - auto codes = url_heads(urls, {}, m_secrets); - Checks::check_exit(VCPKG_LINE_INFO, codes.size() == urls.size()); - for (size_t i = 0; i < codes.size(); ++i) - { - if (codes[i] == 200) - { - cache_status[url_indices[i]]->mark_available(this); - actions_present[url_indices[i]] = CacheAvailability::available; - } - } + urls.push_back(m_url_template.instantiate_variables(BinaryPackageReadInfo{*actions[idx]})); } - for (size_t idx = 0; idx < actions.size(); ++idx) + auto codes = url_heads(urls, {}, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - if (actions_present[idx] == CacheAvailability::unavailable) - { - cache_status[idx]->mark_unavailable(this); - } + out_status[i] = codes[i] == 200 ? CacheAvailability::available : CacheAvailability::unavailable; } } - const VcpkgPaths& paths; - std::vector m_url_templates; + StringView vendor() const override { return "HTTP servers"; } + + Path m_buildtrees; + UrlTemplate m_url_template; std::vector m_secrets; }; - struct NugetBinaryProvider : IBinaryProvider - { - NugetBinaryProvider(const VcpkgPaths& paths, - std::vector&& read_sources, - std::vector&& write_sources, - std::vector&& read_configs, - std::vector&& write_configs, - std::string&& timeout, - bool nuget_interactive) - : paths(paths) - , m_read_sources(std::move(read_sources)) - , m_write_sources(std::move(write_sources)) - , m_read_configs(std::move(read_configs)) - , m_write_configs(std::move(write_configs)) - , m_timeout(std::move(timeout)) - , m_interactive(nuget_interactive) - , m_use_nuget_cache(false) - { - const std::string use_nuget_cache = get_environment_variable("VCPKG_USE_NUGET_CACHE").value_or(""); - m_use_nuget_cache = - Strings::case_insensitive_ascii_equals(use_nuget_cache, "true") || use_nuget_cache == "1"; + + struct NuGetSource + { + StringLiteral option; + std::string value; + }; + + NuGetSource nuget_sources_arg(View sources) { return {"-Source", Strings::join(";", sources)}; } + NuGetSource nuget_configfile_arg(const Path& config_path) { return {"-ConfigFile", config_path.native()}; } + + struct NuGetTool + { + NuGetTool(const ToolCache& cache, MessageSink& sink, const BinaryConfigParserState& shared) + : m_timeout(shared.nugettimeout) + , m_interactive(shared.nuget_interactive) + , m_use_nuget_cache(shared.use_nuget_cache) + { +#ifndef _WIN32 + m_cmd.string_arg(cache.get_tool_path(Tools::MONO, sink)); +#endif + m_cmd.string_arg(cache.get_tool_path(Tools::NUGET, sink)); + } + + ExpectedL push(MessageSink& sink, const Path& nupkg_path, const NuGetSource& src) const + { + return run_nuget_commandline(push_cmd(nupkg_path, src), sink); + } + ExpectedL pack(MessageSink& sink, const Path& nuspec_path, const Path& out_dir) const + { + return run_nuget_commandline(pack_cmd(nuspec_path, out_dir), sink); + } + ExpectedL install(MessageSink& sink, + StringView packages_config, + const Path& out_dir, + const NuGetSource& src) const + { + return run_nuget_commandline(install_cmd(packages_config, out_dir, src), sink); + } + + private: + Command subcommand(StringLiteral sub) const + { + Command cmd = m_cmd; + cmd.string_arg(sub).string_arg("-ForceEnglishOutput").string_arg("-Verbosity").string_arg("detailed"); + if (!m_interactive) cmd.string_arg("-NonInteractive"); + return cmd; + } + + Command install_cmd(StringView packages_config, const Path& out_dir, const NuGetSource& src) const + { + Command cmd = subcommand("install"); + cmd.string_arg(packages_config) + .string_arg("-OutputDirectory") + .string_arg(out_dir) + .string_arg("-ExcludeVersion") + .string_arg("-PreRelease") + .string_arg("-PackageSaveMode") + .string_arg("nupkg"); + if (!m_use_nuget_cache) cmd.string_arg("-DirectDownload").string_arg("-NoCache"); + cmd.string_arg(src.option).string_arg(src.value); + return cmd; + } + + Command pack_cmd(const Path& nuspec_path, const Path& out_dir) const + { + return subcommand("pack") + .string_arg(nuspec_path) + .string_arg("-OutputDirectory") + .string_arg(out_dir) + .string_arg("-NoDefaultExcludes"); + } + + Command push_cmd(const Path& nupkg_path, const NuGetSource& src) const + { + return subcommand("push") + .string_arg(nupkg_path) + .string_arg("-Timeout") + .string_arg(m_timeout) + .string_arg(src.option) + .string_arg(src.value); } ExpectedL run_nuget_commandline(const Command& cmdline, MessageSink& msg_sink) const @@ -663,246 +634,148 @@ namespace }); } - struct NuGetPrefetchAttempt + Command m_cmd; + std::string m_timeout; + bool m_interactive; + bool m_use_nuget_cache; + }; + + struct NugetBaseBinaryProvider + { + NugetBaseBinaryProvider(Filesystem& fs, + const NuGetTool& tool, + const Path& packages, + const Path& buildtrees, + StringView nuget_prefix) + : m_fs(fs) + , m_cmd(tool) + , m_packages(packages) + , m_buildtrees(buildtrees) + , m_nuget_prefix(nuget_prefix.to_string()) + { + } + + Filesystem& m_fs; + NuGetTool m_cmd; + Path m_packages; + Path m_buildtrees; + std::string m_nuget_prefix; + }; + + struct NugetReadBinaryProvider : IReadBinaryProvider, private NugetBaseBinaryProvider + { + NugetReadBinaryProvider(const NugetBaseBinaryProvider& base, NuGetSource src) + : NugetBaseBinaryProvider(base), m_src(std::move(src)) { - PackageSpec spec; - NugetReference reference; - size_t result_index; - }; + } - static void generate_packages_config(Filesystem& fs, - const Path& packages_config, - const std::vector& attempts) + NuGetSource m_src; + + static std::string generate_packages_config(View refs) { XmlSerializer xml; xml.emit_declaration().line_break(); xml.open_tag("packages").line_break(); - for (auto&& attempt : attempts) + for (auto&& ref : refs) { xml.start_complex_open_tag("package") - .text_attr("id", attempt.reference.id) - .text_attr("version", attempt.reference.version) + .text_attr("id", ref.id) + .text_attr("version", ref.version) .finish_self_closing_complex_tag() .line_break(); } xml.close_tag("packages").line_break(); - fs.write_contents(packages_config, xml.buf, VCPKG_LINE_INFO); + return std::move(xml.buf); } - void prefetch(View actions, View cache_status) const override - { - if (m_read_sources.empty() && m_read_configs.empty()) - { - return; - } + // Individual package restore is too expensive with NuGet, so it is not implemented + RestoreResult try_restore(const InstallPlanAction&) const override { return RestoreResult::unavailable; } - const ElapsedTimer timer; - auto& fs = paths.get_filesystem(); + // Prechecking is too expensive with NuGet, so it is not implemented + void precheck(View, Span) const override { } - std::vector attempts; - for (size_t idx = 0; idx < cache_status.size(); ++idx) + StringView vendor() const override { return "NuGet"; } + + void prefetch(View actions, Span out_status) const override + { + auto packages_config = m_buildtrees / "packages.config"; + auto refs = + Util::fmap(actions, [this](const InstallPlanAction* p) { return make_nugetref(*p, m_nuget_prefix); }); + m_fs.write_contents(packages_config, generate_packages_config(refs), VCPKG_LINE_INFO); + m_cmd.install(stdout_sink, packages_config, m_packages, m_src); + for (size_t i = 0; i < actions.size(); ++i) { - auto this_cache_status = cache_status[idx]; - if (!this_cache_status || !this_cache_status->should_attempt_restore(this)) + // nuget.exe provides the nupkg file and the unpacked folder + const auto nupkg_path = m_packages / refs[i].id / refs[i].id + ".nupkg"; + if (m_fs.exists(nupkg_path, IgnoreErrors{})) { - continue; - } + m_fs.remove(nupkg_path, VCPKG_LINE_INFO); + const auto nuget_dir = actions[i]->spec.dir(); + if (nuget_dir != refs[i].id) + { + const auto path_from = m_packages / refs[i].id; + const auto path_to = m_packages / nuget_dir; + m_fs.rename(path_from, path_to, VCPKG_LINE_INFO); + } - const auto& action = actions[idx]; - const auto& spec = action.spec; - fs.remove_all(paths.package_dir(spec), VCPKG_LINE_INFO); - attempts.push_back({spec, make_nugetref(action, get_nuget_prefix()), idx}); + out_status[i] = RestoreResult::restored; + } } + } + }; + + struct NugetBinaryPushProvider : IWriteBinaryProvider, private NugetBaseBinaryProvider + { + NugetBinaryPushProvider(const NugetBaseBinaryProvider& base, + std::vector&& sources, + std::vector&& configs) + : NugetBaseBinaryProvider(base), m_sources(std::move(sources)), m_configs(std::move(configs)) + { + } + + std::vector m_sources; + std::vector m_configs; - if (attempts.empty()) + bool needs_nuspec_data() const override { return true; } + bool needs_zip_file() const override { return false; } + + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override + { + if (!request.nuspec.has_value()) { - return; + Checks::unreachable( + VCPKG_LINE_INFO, + "request.info.nuspec must be non empty because needs_nuspec_data() should return true"); } - msg::println(msgAttemptingToFetchPackagesFromVendor, msg::count = attempts.size(), msg::vendor = "nuget"); + auto& spec = request.spec; - auto packages_config = paths.buildtrees() / "packages.config"; - const auto& nuget_exe = paths.get_tool_exe("nuget", stdout_sink); - std::vector cmdlines; - - if (!m_read_sources.empty()) + auto nuspec_path = m_buildtrees / spec.name() / spec.triplet().canonical_name() + ".nuspec"; + std::error_code ec; + m_fs.write_contents(nuspec_path, request.nuspec.value_or_exit(VCPKG_LINE_INFO), ec); + if (ec) { - // First check using all sources - Command cmdline; -#ifndef _WIN32 - cmdline.string_arg(paths.get_tool_exe(Tools::MONO, stdout_sink)); -#endif - cmdline.string_arg(nuget_exe) - .string_arg("install") - .string_arg(packages_config) - .string_arg("-OutputDirectory") - .string_arg(paths.packages()) - .string_arg("-Source") - .string_arg(Strings::join(";", m_read_sources)) - .string_arg("-ExcludeVersion") - .string_arg("-PreRelease") - .string_arg("-PackageSaveMode") - .string_arg("nupkg") - .string_arg("-Verbosity") - .string_arg("detailed") - .string_arg("-ForceEnglishOutput"); - if (!m_interactive) - { - cmdline.string_arg("-NonInteractive"); - } - if (!m_use_nuget_cache) - { - cmdline.string_arg("-DirectDownload").string_arg("-NoCache"); - } + msg_sink.println(Color::error, msgPackingVendorFailed, msg::vendor = "NuGet"); + return 0; + } - cmdlines.push_back(std::move(cmdline)); + auto packed_result = m_cmd.pack(msg_sink, nuspec_path, m_buildtrees); + m_fs.remove(nuspec_path, IgnoreErrors{}); + if (!packed_result) + { + msg_sink.println(Color::error, msgPackingVendorFailed, msg::vendor = "NuGet"); + return 0; } - for (auto&& cfg : m_read_configs) + int count_stored = 0; + auto nupkg_path = m_buildtrees / make_nugetref(request, m_nuget_prefix).nupkg_filename(); + for (auto&& write_src : m_sources) { - // Then check using each config - Command cmdline; -#ifndef _WIN32 - cmdline.string_arg(paths.get_tool_exe(Tools::MONO, stdout_sink)); -#endif - cmdline.string_arg(nuget_exe) - .string_arg("install") - .string_arg(packages_config) - .string_arg("-OutputDirectory") - .string_arg(paths.packages()) - .string_arg("-ConfigFile") - .string_arg(cfg) - .string_arg("-ExcludeVersion") - .string_arg("-PreRelease") - .string_arg("-PackageSaveMode") - .string_arg("nupkg") - .string_arg("-Verbosity") - .string_arg("detailed") - .string_arg("-ForceEnglishOutput"); - if (!m_interactive) - { - cmdline.string_arg("-NonInteractive"); - } - if (!m_use_nuget_cache) - { - cmdline.string_arg("-DirectDownload").string_arg("-NoCache"); - } - - cmdlines.push_back(std::move(cmdline)); - } - - const size_t total_restore_attempts = attempts.size(); - for (const auto& cmdline : cmdlines) - { - if (attempts.empty()) - { - break; - } - - generate_packages_config(fs, packages_config, attempts); - run_nuget_commandline(cmdline, stdout_sink); - Util::erase_remove_if(attempts, [&](const NuGetPrefetchAttempt& nuget_ref) -> bool { - // note that we would like the nupkg downloaded to buildtrees, but nuget.exe downloads it to the - // output directory - const auto nupkg_path = - paths.packages() / nuget_ref.reference.id / nuget_ref.reference.id + ".nupkg"; - if (fs.exists(nupkg_path, IgnoreErrors{})) - { - fs.remove(nupkg_path, VCPKG_LINE_INFO); - const auto nuget_dir = nuget_ref.spec.dir(); - if (nuget_dir != nuget_ref.reference.id) - { - const auto path_from = paths.packages() / nuget_ref.reference.id; - const auto path_to = paths.packages() / nuget_dir; - fs.rename(path_from, path_to, VCPKG_LINE_INFO); - } - - cache_status[nuget_ref.result_index]->mark_restored(); - return true; - } - - return false; - }); - } - msg::println(msgRestoredPackagesFromVendor, - msg::count = total_restore_attempts - attempts.size(), - msg::elapsed = timer.elapsed(), - msg::value = "NuGet"); - } - - RestoreResult try_restore(const InstallPlanAction&) const override { return RestoreResult::unavailable; } - - bool needs_nuspec_data() const override { return !m_write_sources.empty() || !m_write_configs.empty(); } - - size_t push_success(const BinaryProviderPushRequest& request, MessageSink& msg_sink) override - { - if (m_write_sources.empty() && m_write_configs.empty()) - { - return 0; - } - if (!request.info.nuspec.has_value()) - { - Checks::unreachable( - VCPKG_LINE_INFO, - "request.info.nuspec must be non empty because needs_nuspec_data() should return true"); - } - - auto& spec = request.info.spec; - - NugetReference nuget_ref = make_nugetref(request.info, get_nuget_prefix()); - auto nuspec_path = paths.buildtrees() / spec.name() / (spec.triplet().to_string() + ".nuspec"); - auto& fs = paths.get_filesystem(); - fs.write_contents(nuspec_path, request.info.nuspec.value_or_exit(VCPKG_LINE_INFO), VCPKG_LINE_INFO); - - const auto& nuget_exe = paths.get_tool_exe("nuget", stdout_sink); - Command cmdline; -#ifndef _WIN32 - cmdline.string_arg(paths.get_tool_exe(Tools::MONO, stdout_sink)); -#endif - cmdline.string_arg(nuget_exe) - .string_arg("pack") - .string_arg(nuspec_path) - .string_arg("-OutputDirectory") - .string_arg(paths.buildtrees()) - .string_arg("-NoDefaultExcludes") - .string_arg("-ForceEnglishOutput"); - - if (!m_interactive) - { - cmdline.string_arg("-NonInteractive"); - } - - if (!run_nuget_commandline(cmdline, msg_sink)) - { - msg_sink.println(Color::error, msgPackingVendorFailed, msg::vendor = "NuGet"); - return 0; - } - int count_stored = 0; - auto nupkg_path = paths.buildtrees() / nuget_ref.nupkg_filename(); - for (auto&& write_src : m_write_sources) - { - Command cmd; -#ifndef _WIN32 - cmd.string_arg(paths.get_tool_exe(Tools::MONO, stdout_sink)); -#endif - cmd.string_arg(nuget_exe) - .string_arg("push") - .string_arg(nupkg_path) - .string_arg("-ForceEnglishOutput") - .string_arg("-Timeout") - .string_arg(m_timeout) - .string_arg("-Source") - .string_arg(write_src); - - if (!m_interactive) - { - cmd.string_arg("-NonInteractive"); - } msg_sink.println( msgUploadingBinariesToVendor, msg::spec = spec, msg::vendor = "NuGet", msg::path = write_src); - if (!run_nuget_commandline(cmd, msg_sink)) + if (!m_cmd.push(msg_sink, nupkg_path, nuget_sources_arg({&write_src, 1}))) { msg_sink.println( Color::error, msgPushingVendorFailed, msg::vendor = "NuGet", msg::path = write_src); @@ -912,33 +785,16 @@ namespace count_stored++; } } - for (auto&& write_cfg : m_write_configs) + for (auto&& write_cfg : m_configs) { - Command cmd; -#ifndef _WIN32 - cmd.string_arg(paths.get_tool_exe(Tools::MONO, msg_sink)); -#endif - cmd.string_arg(nuget_exe) - .string_arg("push") - .string_arg(nupkg_path) - .string_arg("-ForceEnglishOutput") - .string_arg("-Timeout") - .string_arg(m_timeout) - .string_arg("-ConfigFile") - .string_arg(write_cfg); - if (!m_interactive) - { - cmd.string_arg("-NonInteractive"); - } - msg_sink.println(Color::error, - msgUploadingBinariesUsingVendor, + msg_sink.println(msgUploadingBinariesToVendor, msg::spec = spec, msg::vendor = "NuGet config", msg::path = write_cfg); - if (!run_nuget_commandline(cmd, msg_sink)) + if (!m_cmd.push(msg_sink, nupkg_path, nuget_configfile_arg(write_cfg))) { msg_sink.println( - Color::error, msgPushingVendorFailed, msg::vendor = "NuGet", msg::path = write_cfg); + Color::error, msgPushingVendorFailed, msg::vendor = "NuGet config", msg::path = write_cfg); } else { @@ -946,37 +802,20 @@ namespace } } - fs.remove(nupkg_path, IgnoreErrors{}); + m_fs.remove(nupkg_path, IgnoreErrors{}); return count_stored; } - - void precheck(View, View) const override { } - - private: - const VcpkgPaths& paths; - - std::vector m_read_sources; - std::vector m_write_sources; - - std::vector m_read_configs; - std::vector m_write_configs; - - std::string m_timeout; - bool m_interactive; - bool m_use_nuget_cache; }; - struct GHABinaryProvider : IBinaryProvider + + struct GHABinaryProvider : ZipReadBinaryProvider { - GHABinaryProvider(const VcpkgPaths& paths, - bool read, - bool write, - const Optional& url, - const Optional& token) - : paths(paths) + GHABinaryProvider( + ZipTool zip, Filesystem& fs, const Path& buildtrees, const std::string& url, const std::string& token) + : ZipReadBinaryProvider(std::move(zip), fs) + , m_buildtrees(buildtrees) + , m_url(url + "_apis/artifactcache/cache") + , m_token_header("Authorization: Bearer " + token) { - if (read) m_read_url = url.value_or_exit(VCPKG_LINE_INFO) + "_apis/artifactcache/cache"; - if (write) m_write_url = url.value_or_exit(VCPKG_LINE_INFO) + "_apis/artifactcache/caches"; - m_token_header = "Authorization: Bearer " + token.value_or_exit(VCPKG_LINE_INFO); } Command command() const @@ -996,7 +835,7 @@ namespace std::string lookup_cache_entry(const std::string& abi) const { auto cmd = command() - .string_arg(m_read_url) + .string_arg(m_url) .string_arg("-G") .string_arg("-d") .string_arg("keys=vcpkg") @@ -1011,195 +850,154 @@ namespace return json.get()->get("archiveLocation")->string(VCPKG_LINE_INFO).to_string(); } - Optional reserve_cache_entry(const std::string& abi, int64_t cacheSize) const - { - Json::Object payload; - payload.insert("key", "vcpkg"); - payload.insert("version", abi); - payload.insert("cacheSize", Json::Value::integer(cacheSize)); - auto cmd = command().string_arg(m_write_url).string_arg("-d").string_arg(stringify(payload)); - - auto res = cmd_execute_and_capture_output(cmd); - if (!res.has_value() || res.get()->exit_code) return {}; - auto json = Json::parse_object(res.get()->output); - if (!json.has_value() || !json.get()->contains("cacheId")) return {}; - return json.get()->get("cacheId")->integer(VCPKG_LINE_INFO); - } - - void prefetch(View actions, View cache_status) const override + void acquire_zips(View actions, Span> out_zip_paths) const override { - auto& fs = paths.get_filesystem(); - - const ElapsedTimer timer; - size_t restored_count = 0; std::vector> url_paths; std::vector url_indices; - if (!m_read_url.empty()) + for (size_t idx = 0; idx < actions.size(); ++idx) { - for (size_t idx = 0; idx < cache_status.size(); ++idx) - { - const auto this_cache_status = cache_status[idx]; - if (!this_cache_status || !this_cache_status->should_attempt_restore(this)) - { - continue; - } - - auto&& action = actions[idx]; - auto url = lookup_cache_entry(action.package_abi().value_or_exit(VCPKG_LINE_INFO)); - if (url.empty()) continue; + auto&& action = *actions[idx]; + auto url = lookup_cache_entry(action.package_abi().value_or_exit(VCPKG_LINE_INFO)); + if (url.empty()) continue; - clean_prepare_dir(fs, paths.package_dir(action.spec)); - url_paths.emplace_back(std::move(url), make_temp_archive_path(paths.buildtrees(), action.spec)); - url_indices.push_back(idx); - } + url_paths.emplace_back(std::move(url), make_temp_archive_path(m_buildtrees, action.spec)); + url_indices.push_back(idx); } - if (!url_paths.empty()) - { - msg::println( - msgAttemptingToFetchPackagesFromVendor, msg::count = url_paths.size(), msg::vendor = "GHA"); + auto codes = download_files(m_fs, url_paths, {}); - auto codes = download_files(fs, url_paths, {}); - std::vector action_idxs; - std::vector jobs; - for (size_t i = 0; i < codes.size(); ++i) - { - if (codes[i] == 200) - { - action_idxs.push_back(i); - jobs.push_back(decompress_zip_archive_cmd(paths.get_tool_cache(), - stdout_sink, - paths.package_dir(actions[url_indices[i]].spec), - url_paths[i].second)); - } - } - auto job_results = decompress_in_parallel(jobs); - for (size_t j = 0; j < jobs.size(); ++j) + for (size_t i = 0; i < codes.size(); ++i) + { + if (codes[i] == 200) { - const auto i = action_idxs[j]; - if (job_results[j]) - { - ++restored_count; - fs.remove(url_paths[i].second, VCPKG_LINE_INFO); - cache_status[url_indices[i]]->mark_restored(); - } - else - { - Debug::print("Failed to decompress ", url_paths[i].second, '\n'); - } + out_zip_paths[url_indices[i]] = url_paths[i].second; } } + } - msg::println(msgRestoredPackagesFromVendor, - msg::count = restored_count, - msg::elapsed = timer.elapsed(), - msg::value = "GHA"); + void post_decompress(const InstallPlanAction&, const Path& archive_path, bool) const override + { + m_fs.remove(archive_path, VCPKG_LINE_INFO); } - RestoreResult try_restore(const InstallPlanAction&) const override { return RestoreResult::unavailable; } + void precheck(View, Span) const override { } + + StringView vendor() const override { return "GitHub Actions Cache"; } + + static constexpr StringLiteral m_accept_header = "Accept: application/json;api-version=6.0-preview.1"; + + Path m_buildtrees; + std::string m_url; + std::string m_token_header; + }; + + struct GHABinaryPushProvider : IWriteBinaryProvider + { + GHABinaryPushProvider(const Filesystem& fs, const std::string& url, const std::string& token) + : m_fs(fs), m_url(url + "_apis/artifactcache/caches"), m_token_header("Authorization: Bearer " + token) + { + } + + Command command() const + { + Command cmd; + cmd.string_arg("curl") + .string_arg("-s") + .string_arg("-H") + .string_arg("Content-Type: application/json") + .string_arg("-H") + .string_arg(m_token_header) + .string_arg("-H") + .string_arg(m_accept_header); + return cmd; + } + + Optional reserve_cache_entry(const std::string& abi, int64_t cacheSize) const + { + Json::Object payload; + payload.insert("key", "vcpkg"); + payload.insert("version", abi); + payload.insert("cacheSize", Json::Value::integer(cacheSize)); + auto cmd = command().string_arg(m_url).string_arg("-d").string_arg(stringify(payload)); + + auto res = cmd_execute_and_capture_output(cmd); + if (!res.has_value() || res.get()->exit_code) return {}; + auto json = Json::parse_object(res.get()->output); + if (!json.has_value() || !json.get()->contains("cacheId")) return {}; + return json.get()->get("cacheId")->integer(VCPKG_LINE_INFO); + } - size_t push_success(const BinaryProviderPushRequest& request, MessageSink& msg_sink) override + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink&) override { - if (m_write_url.empty()) return 0; + if (!request.zip_path) return 0; + const auto& zip_path = *request.zip_path.get(); const ElapsedTimer timer; - auto& fs = paths.get_filesystem(); - const auto& abi = request.info.package_abi; - auto& spec = request.info.spec; - const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec); - auto compression_result = compress_directory_to_zip( - paths.get_filesystem(), paths.get_tool_cache(), msg_sink, paths.package_dir(spec), tmp_archive_path); - if (!compression_result) - { - msg_sink.println(Color::warning, - msg::format_warning(msgCompressFolderFailed, msg::path = paths.package_dir(spec)) - .append_raw(' ') - .append_raw(compression_result.error())); - return 0; - } + const auto& abi = request.package_abi; int64_t cache_size; { - auto archive = fs.open_for_read(tmp_archive_path, VCPKG_LINE_INFO); + auto archive = m_fs.open_for_read(zip_path, VCPKG_LINE_INFO); archive.try_seek_to(0, SEEK_END); cache_size = archive.tell(); } size_t upload_count = 0; - if (!m_write_url.empty()) - { - if (auto cacheId = reserve_cache_entry(abi, cache_size)) - { - std::vector headers{ - m_token_header, - m_accept_header.to_string(), - "Content-Type: application/octet-stream", - "Content-Range: bytes 0-" + std::to_string(cache_size) + "/*", - }; - auto url = m_write_url + "/" + std::to_string(*cacheId.get()); - if (put_file(fs, url, {}, headers, tmp_archive_path, "PATCH")) + if (auto cacheId = reserve_cache_entry(abi, cache_size)) + { + std::vector headers{ + m_token_header, + m_accept_header.to_string(), + "Content-Type: application/octet-stream", + "Content-Range: bytes 0-" + std::to_string(cache_size) + "/*", + }; + auto url = m_url + "/" + std::to_string(*cacheId.get()); + if (put_file(m_fs, url, {}, headers, zip_path, "PATCH")) + { + Json::Object commit; + commit.insert("size", std::to_string(cache_size)); + auto cmd = command().string_arg(url).string_arg("-d").string_arg(stringify(commit)); + + auto res = cmd_execute_and_capture_output(cmd); + if (res.has_value() && !res.get()->exit_code) { - Json::Object commit; - commit.insert("size", std::to_string(cache_size)); - auto cmd = command().string_arg(url).string_arg("-d").string_arg(stringify(commit)); - - auto res = cmd_execute_and_capture_output(cmd); - if (res.has_value() && !res.get()->exit_code) - { - ++upload_count; - } + ++upload_count; } } } return upload_count; } - void precheck(View actions, View cache_status) const override - { - std::vector actions_availability{actions.size()}; - if (!m_read_url.empty()) - { - for (size_t idx = 0; idx < actions.size(); ++idx) - { - auto&& action = actions[idx]; - const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); - if (!cache_status[idx]->should_attempt_precheck(this)) - { - continue; - } - - if (!lookup_cache_entry(abi).empty()) - { - actions_availability[idx] = CacheAvailability::available; - cache_status[idx]->mark_available(this); - } - } - } - - for (size_t idx = 0; idx < actions.size(); ++idx) - { - const auto this_cache_status = cache_status[idx]; - if (this_cache_status && actions_availability[idx] == CacheAvailability::unavailable) - { - this_cache_status->mark_unavailable(this); - } - } - } + bool needs_nuspec_data() const override { return false; } + bool needs_zip_file() const override { return true; } static constexpr StringLiteral m_accept_header = "Accept: application/json;api-version=6.0-preview.1"; + + const Filesystem& m_fs; + std::string m_url; std::string m_token_header; + }; - std::string m_read_url; - std::string m_write_url; + struct IObjectStorageTool + { + virtual ~IObjectStorageTool() = default; - const VcpkgPaths& paths; + virtual StringLiteral vendor() const = 0; + virtual ExpectedL stat(StringView url) const = 0; + virtual ExpectedL download_file(StringView object, const Path& archive) const = 0; + virtual ExpectedL upload_file(StringView object, const Path& archive) const = 0; }; - struct ObjectStorageProvider : IBinaryProvider + struct ObjectStorageProvider : ZipReadBinaryProvider { - ObjectStorageProvider(const VcpkgPaths& paths, - std::vector&& read_prefixes, - std::vector&& write_prefixes) - : paths(paths), m_read_prefixes(std::move(read_prefixes)), m_write_prefixes(std::move(write_prefixes)) + ObjectStorageProvider(ZipTool zip, + Filesystem& fs, + const Path& buildtrees, + std::string&& prefix, + std::shared_ptr tool) + : ZipReadBinaryProvider(std::move(zip), fs) + , m_buildtrees(buildtrees) + , m_prefix(std::move(prefix)) + , m_tool(std::move(tool)) { } @@ -1208,213 +1006,133 @@ namespace return Strings::concat(prefix, abi, ".zip"); } - void prefetch(View actions, View cache_status) const override + void acquire_zips(View actions, Span> out_zip_paths) const override { - auto& fs = paths.get_filesystem(); - - const ElapsedTimer timer; - size_t restored_count = 0; - for (const auto& prefix : m_read_prefixes) + for (size_t idx = 0; idx < actions.size(); ++idx) { - std::vector> url_paths; - std::vector url_indices; - - for (size_t idx = 0; idx < cache_status.size(); ++idx) + auto&& action = *actions[idx]; + const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); + auto tmp = make_temp_archive_path(m_buildtrees, action.spec); + auto res = m_tool->download_file(make_object_path(m_prefix, abi), tmp); + if (res) { - const auto this_cache_status = cache_status[idx]; - if (!this_cache_status || !this_cache_status->should_attempt_restore(this)) - { - continue; - } - - auto&& action = actions[idx]; - clean_prepare_dir(fs, paths.package_dir(action.spec)); - url_paths.emplace_back( - make_object_path(prefix, action.package_abi().value_or_exit(VCPKG_LINE_INFO)), - make_temp_archive_path(paths.buildtrees(), action.spec)); - url_indices.push_back(idx); + out_zip_paths[idx] = std::move(tmp); } - - if (url_paths.empty()) break; - - msg::println( - msgAttemptingToFetchPackagesFromVendor, msg::count = url_paths.size(), msg::vendor = vendor()); - - std::vector jobs; - std::vector idxs; - for (size_t idx = 0; idx < url_paths.size(); ++idx) + else { - auto&& action = actions[url_indices[idx]]; - auto&& url_path = url_paths[idx]; - if (!download_file(url_path.first, url_path.second)) continue; - jobs.push_back(decompress_zip_archive_cmd( - paths.get_tool_cache(), stdout_sink, paths.package_dir(action.spec), url_path.second)); - idxs.push_back(idx); + stdout_sink.println_warning(res.error()); } + } + } - const auto job_results = - cmd_execute_and_capture_output_parallel(jobs, default_working_directory, get_clean_environment()); + virtual void post_decompress(const InstallPlanAction&, const Path& archive_path, bool) const + { + m_fs.remove(archive_path, IgnoreErrors{}); + } - for (size_t j = 0; j < jobs.size(); ++j) + void precheck(View actions, Span cache_status) const override + { + for (size_t idx = 0; idx < actions.size(); ++idx) + { + auto&& action = *actions[idx]; + const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); + if (m_tool->stat(make_object_path(m_prefix, abi))) { - const auto idx = idxs[j]; - if (!job_results[j]) - { - Debug::print("Failed to decompress ", url_paths[idx].second, '\n'); - continue; - } - - // decompression success - ++restored_count; - fs.remove(url_paths[idx].second, VCPKG_LINE_INFO); - cache_status[url_indices[idx]]->mark_restored(); + cache_status[idx] = CacheAvailability::available; + } + else + { + cache_status[idx] = CacheAvailability::unavailable; } } - - msg::println(msgRestoredPackagesFromVendor, - msg::count = restored_count, - msg::elapsed = timer.elapsed(), - msg::value = vendor()); } - RestoreResult try_restore(const InstallPlanAction&) const override { return RestoreResult::unavailable; } + StringView vendor() const override { return m_tool->vendor(); } - size_t push_success(const BinaryProviderPushRequest& request, MessageSink& msg_sink) override + Path m_buildtrees; + std::string m_prefix; + std::shared_ptr m_tool; + }; + struct ObjectStoragePushProvider : IWriteBinaryProvider + { + ObjectStoragePushProvider(std::vector&& prefixes, std::shared_ptr tool) + : m_prefixes(std::move(prefixes)), m_tool(std::move(tool)) { - if (m_write_prefixes.empty()) return 0; - const ElapsedTimer timer; - const auto& abi = request.info.package_abi; - auto& spec = request.info.spec; - const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec); - auto compression_result = compress_directory_to_zip( - paths.get_filesystem(), paths.get_tool_cache(), msg_sink, request.package_dir, tmp_archive_path); - if (!compression_result) - { - msg_sink.println(Color::warning, - msg::format_warning(msgCompressFolderFailed, msg::path = request.package_dir) - .append_raw(' ') - .append_raw(compression_result.error())); - return 0; - } + } - size_t upload_count = 0; - for (const auto& prefix : m_write_prefixes) - { - if (upload_file(make_object_path(prefix, abi), tmp_archive_path)) - { - ++upload_count; - } - } - return upload_count; + static std::string make_object_path(const std::string& prefix, const std::string& abi) + { + return Strings::concat(prefix, abi, ".zip"); } - void precheck(View actions, View cache_status) const override + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override { - std::vector actions_availability{actions.size()}; - for (const auto& prefix : m_read_prefixes) + if (!request.zip_path) return 0; + const auto& zip_path = *request.zip_path.get(); + size_t upload_count = 0; + for (const auto& prefix : m_prefixes) { - for (size_t idx = 0; idx < actions.size(); ++idx) + auto res = m_tool->upload_file(make_object_path(prefix, request.package_abi), zip_path); + if (res) { - auto&& action = actions[idx]; - const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO); - if (!cache_status[idx]->should_attempt_precheck(this)) - { - continue; - } - - if (stat(make_object_path(prefix, abi))) - { - actions_availability[idx] = CacheAvailability::available; - cache_status[idx]->mark_available(this); - } + ++upload_count; } - } - - for (size_t idx = 0; idx < actions.size(); ++idx) - { - const auto this_cache_status = cache_status[idx]; - if (this_cache_status && actions_availability[idx] == CacheAvailability::unavailable) + else { - this_cache_status->mark_unavailable(this); + msg_sink.println_warning(res.error()); } } + return upload_count; } - protected: - virtual StringLiteral vendor() const = 0; - virtual bool stat(StringView url) const = 0; - virtual bool upload_file(StringView object, const Path& archive) const = 0; - virtual bool download_file(StringView object, const Path& archive) const = 0; - - const VcpkgPaths& paths; + bool needs_nuspec_data() const override { return false; } + bool needs_zip_file() const override { return true; } - private: - std::vector m_read_prefixes; - std::vector m_write_prefixes; + std::vector m_prefixes; + std::shared_ptr m_tool; }; - struct GcsBinaryProvider : ObjectStorageProvider + struct GcsStorageTool : IObjectStorageTool { - GcsBinaryProvider(const VcpkgPaths& paths, - std::vector&& read_prefixes, - std::vector&& write_prefixes) - : ObjectStorageProvider(paths, std::move(read_prefixes), std::move(write_prefixes)) - { - } + GcsStorageTool(const ToolCache& cache, MessageSink& sink) : m_tool(cache.get_tool_path(Tools::GSUTIL, sink)) { } StringLiteral vendor() const override { return "GCS"; } - Command command() const { return Command{paths.get_tool_exe(Tools::GSUTIL, stdout_sink)}; } + Command command() const { return m_tool; } - bool stat(StringView url) const override + ExpectedL stat(StringView url) const override { auto cmd = command().string_arg("-q").string_arg("stat").string_arg(url); - return succeeded(cmd_execute(cmd)); + return flatten(cmd_execute_and_capture_output(cmd), Tools::GSUTIL); } - bool upload_file(StringView object, const Path& archive) const override + ExpectedL download_file(StringView object, const Path& archive) const override { - auto cmd = command().string_arg("-q").string_arg("cp").string_arg(archive).string_arg(object); - const auto out = flatten(cmd_execute_and_capture_output(cmd), Tools::GSUTIL); - if (out) - { - return true; - } - - msg::write_unlocalized_text_to_stdout(Color::warning, out.error()); - return false; + auto cmd = command().string_arg("-q").string_arg("cp").string_arg(object).string_arg(archive); + return flatten(cmd_execute_and_capture_output(cmd), Tools::GSUTIL); } - bool download_file(StringView object, const Path& archive) const override + ExpectedL upload_file(StringView object, const Path& archive) const override { - auto cmd = command().string_arg("-q").string_arg("cp").string_arg(object).string_arg(archive); - const auto out = flatten(cmd_execute_and_capture_output(cmd), Tools::GSUTIL); - if (out) - { - return true; - } - - msg::write_unlocalized_text_to_stdout(Color::warning, out.error()); - return false; + auto cmd = command().string_arg("-q").string_arg("cp").string_arg(archive).string_arg(object); + return flatten(cmd_execute_and_capture_output(cmd), Tools::GSUTIL); } + + Command m_tool; }; - struct AwsBinaryProvider : ObjectStorageProvider + struct AwsStorageTool : IObjectStorageTool { - AwsBinaryProvider(const VcpkgPaths& paths, - std::vector&& read_prefixes, - std::vector&& write_prefixes, - const bool no_sign_request) - : ObjectStorageProvider(paths, std::move(read_prefixes), std::move(write_prefixes)) - , m_no_sign_request(no_sign_request) + AwsStorageTool(const ToolCache& cache, MessageSink& sink, bool no_sign_request) + : m_tool(cache.get_tool_path(Tools::AWSCLI, sink)), m_no_sign_request(no_sign_request) { } StringLiteral vendor() const override { return "AWS"; } - Command command() const { return Command{paths.get_tool_exe(Tools::AWSCLI, stdout_sink)}; } + Command command() const { return m_tool; } - bool stat(StringView url) const override + ExpectedL stat(StringView url) const override { auto cmd = command().string_arg("s3").string_arg("ls").string_arg(url); if (m_no_sign_request) @@ -1422,446 +1140,66 @@ namespace cmd.string_arg("--no-sign-request"); } - return succeeded(cmd_execute(cmd)); + return flatten(cmd_execute_and_capture_output(cmd), Tools::AWSCLI); } - bool upload_file(StringView object, const Path& archive) const override + ExpectedL download_file(StringView object, const Path& archive) const override { - auto cmd = command().string_arg("s3").string_arg("cp").string_arg(archive).string_arg(object); + auto r = stat(object); + if (!r) return r; + + auto cmd = command().string_arg("s3").string_arg("cp").string_arg(object).string_arg(archive); if (m_no_sign_request) { cmd.string_arg("--no-sign-request"); } - const auto out = flatten(cmd_execute_and_capture_output(cmd), Tools::AWSCLI); - if (out) - { - return true; - } - msg::write_unlocalized_text_to_stdout(Color::warning, out.error()); - return false; + return flatten(cmd_execute_and_capture_output(cmd), Tools::AWSCLI); } - bool download_file(StringView object, const Path& archive) const override + ExpectedL upload_file(StringView object, const Path& archive) const override { - if (!stat(object)) - { - return false; - } - - auto cmd = command().string_arg("s3").string_arg("cp").string_arg(object).string_arg(archive); + auto cmd = command().string_arg("s3").string_arg("cp").string_arg(archive).string_arg(object); if (m_no_sign_request) { cmd.string_arg("--no-sign-request"); } - - const auto out = flatten(cmd_execute_and_capture_output(cmd), Tools::AWSCLI); - if (out) - { - return true; - } - - msg::write_unlocalized_text_to_stdout(Color::warning, out.error()); - return false; + return flatten(cmd_execute_and_capture_output(cmd), Tools::AWSCLI); } - private: + Command m_tool; bool m_no_sign_request; }; - struct CosBinaryProvider : ObjectStorageProvider + struct CosStorageTool : IObjectStorageTool { - CosBinaryProvider(const VcpkgPaths& paths, - std::vector&& read_prefixes, - std::vector&& write_prefixes) - : ObjectStorageProvider(paths, std::move(read_prefixes), std::move(write_prefixes)) - { - } + CosStorageTool(const ToolCache& cache, MessageSink& sink) : m_tool(cache.get_tool_path(Tools::COSCLI, sink)) { } StringLiteral vendor() const override { return "COS"; } - Command command() const { return Command{paths.get_tool_exe(Tools::COSCLI, stdout_sink)}; } + Command command() const { return m_tool; } - bool stat(StringView url) const override + ExpectedL stat(StringView url) const override { auto cmd = command().string_arg("ls").string_arg(url); - return succeeded(cmd_execute(cmd)); + return flatten(cmd_execute_and_capture_output(cmd), Tools::COSCLI); } - bool upload_file(StringView object, const Path& archive) const override - { - auto cmd = command().string_arg("cp").string_arg(archive).string_arg(object); - const auto out = flatten(cmd_execute_and_capture_output(cmd), Tools::COSCLI); - if (out) - { - return true; - } - - msg::write_unlocalized_text_to_stdout(Color::warning, out.error()); - return false; - } - - bool download_file(StringView object, const Path& archive) const override + ExpectedL download_file(StringView object, const Path& archive) const override { auto cmd = command().string_arg("cp").string_arg(object).string_arg(archive); - const auto out = flatten(cmd_execute_and_capture_output(cmd), Tools::COSCLI); - if (out) - { - return true; - } - - msg::write_unlocalized_text_to_stdout(Color::warning, out.error()); - return false; - } - }; -} - -namespace vcpkg -{ - LocalizedString UrlTemplate::valid() const - { - std::vector invalid_keys; - auto result = api_stable_format(url_template, [&](std::string&, StringView key) { - static constexpr std::array valid_keys = {"name", "version", "sha", "triplet"}; - if (!Util::Vectors::contains(valid_keys, key)) - { - invalid_keys.push_back(key.to_string()); - } - }); - if (!result) - { - return result.error(); - } - if (!invalid_keys.empty()) - { - return msg::format(msgUnknownVariablesInTemplate, - msg::value = url_template, - msg::list = Strings::join(", ", invalid_keys)); - } - return {}; - } - - std::string UrlTemplate::instantiate_variables(const BinaryPackageInformation& info) const - { - return api_stable_format(url_template, - [&](std::string& out, StringView key) { - if (key == "version") - { - out += info.raw_version; - } - else if (key == "name") - { - out += info.spec.name(); - } - else if (key == "triplet") - { - out += info.spec.triplet().canonical_name(); - } - else if (key == "sha") - { - out += info.package_abi; - } - else - { - Debug::println("Unknown key: ", key); - // We do a input validation while parsing the config - Checks::unreachable(VCPKG_LINE_INFO); - }; - }) - .value_or_exit(VCPKG_LINE_INFO); - } - - BinaryCache::BinaryCache(Filesystem& filesystem) : filesystem(filesystem) { } - - BinaryCache::BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths) - : BinaryCache(paths.get_filesystem()) - { - install_providers_for(args, paths); - } - - void BinaryCache::install_providers(std::vector>&& providers) - { - Checks::check_exit( - VCPKG_LINE_INFO, m_status.empty(), "Attempted to install additional providers in active binary cache"); - if (m_providers.empty()) - { - m_providers = std::move(providers); - } - else - { - m_providers.insert(m_providers.end(), - std::make_move_iterator(providers.begin()), - std::make_move_iterator(providers.end())); - } - needs_nuspec_data = Util::any_of(m_providers, [](auto& provider) { return provider->needs_nuspec_data(); }); - } - - void BinaryCache::install_providers_for(const VcpkgCmdArguments& args, const VcpkgPaths& paths) - { - if (args.binary_caching_enabled()) - { - install_providers( - create_binary_providers_from_configs(paths, args.binary_sources).value_or_exit(VCPKG_LINE_INFO)); - } - } - - RestoreResult BinaryCache::try_restore(const InstallPlanAction& action) - { - const auto abi = action.package_abi().get(); - if (!abi) - { - // e.g. this is a `--head` package - return RestoreResult::unavailable; - } - - auto& cache_status = m_status[*abi]; - if (cache_status.is_restored()) - { - return RestoreResult::restored; - } - - const auto available = cache_status.get_available_provider(); - if (available) - { - switch (available->try_restore(action)) - { - case RestoreResult::unavailable: - // Even though that provider thought it had it, it didn't; perhaps - // due to intermittent network problems etc. - // Try other providers below - break; - case RestoreResult::restored: cache_status.mark_restored(); return RestoreResult::restored; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - for (auto&& provider : m_providers) - { - if (provider.get() == available) - { - continue; // this one already tried :) - } - - if (cache_status.is_unavailable(m_providers.size())) - { - break; - } - - switch (provider->try_restore(action)) - { - case RestoreResult::restored: cache_status.mark_restored(); return RestoreResult::restored; - case RestoreResult::unavailable: cache_status.mark_unavailable(provider.get()); break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - return RestoreResult::unavailable; - } - - void BinaryCache::push_success(const InstallPlanAction& action, Path package_dir) - { - if (action.package_abi().has_value()) - { - std::string nuspec; - if (needs_nuspec_data) - { - NugetReference nuget_ref = make_nugetref(action, get_nuget_prefix()); - nuspec = generate_nuspec(package_dir, action, nuget_ref); - } - size_t num_destinations = 0; - BinaryProviderPushRequest request{BinaryPackageInformation{action, std::move(nuspec)}, package_dir}; - for (auto&& provider : m_providers) - { - num_destinations += provider->push_success(request, stdout_sink); - } - stdout_sink.println(msgStoredBinariesToDestinations, msg::count = num_destinations); - } - if (action.build_options.clean_packages == CleanPackages::YES) - { - filesystem.remove_all(package_dir, VCPKG_LINE_INFO); - } - } - - void BinaryCache::prefetch(View actions) - { - std::vector cache_status{actions.size()}; - for (size_t idx = 0; idx < actions.size(); ++idx) - { - const auto abi = actions[idx].package_abi().get(); - if (abi) - { - cache_status[idx] = &m_status[*abi]; - } - } - - for (auto&& provider : m_providers) - { - provider->prefetch(actions, cache_status); - for (auto status : cache_status) - { - if (status) - { - status->mark_unavailable(provider.get()); - } - } - } - } - - std::vector BinaryCache::precheck(View actions) - { - std::vector cache_status{actions.size()}; - for (size_t idx = 0; idx < actions.size(); ++idx) - { - auto& action = actions[idx]; - const auto abi = action.package_abi().get(); - if (!abi) - { - Checks::unreachable(VCPKG_LINE_INFO, fmt::format("{} did not have an ABI", action.spec)); - } - - cache_status[idx] = &m_status[*abi]; - } - - for (auto&& provider : m_providers) - { - provider->precheck(actions, cache_status); - } - - std::vector results{actions.size()}; - for (size_t idx = 0; idx < results.size(); ++idx) - { - results[idx] = cache_status[idx]->get_available_provider() ? CacheAvailability::available - : CacheAvailability::unavailable; - } - - return results; - } - - bool CacheStatus::should_attempt_precheck(const IBinaryProvider* sender) const noexcept - { - switch (m_status) - { - case CacheStatusState::unknown: return !Util::Vectors::contains(m_known_unavailable_providers, sender); - case CacheStatusState::available: return false; - case CacheStatusState::restored: return false; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - bool CacheStatus::should_attempt_restore(const IBinaryProvider* sender) const noexcept - { - switch (m_status) - { - case CacheStatusState::unknown: return !Util::Vectors::contains(m_known_unavailable_providers, sender); - case CacheStatusState::available: return m_available_provider == sender; - case CacheStatusState::restored: return false; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - bool CacheStatus::is_unavailable(size_t total_providers) const noexcept - { - switch (m_status) - { - case CacheStatusState::unknown: return m_known_unavailable_providers.size() <= total_providers; - case CacheStatusState::available: - case CacheStatusState::restored: return false; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - bool CacheStatus::is_restored() const noexcept { return m_status == CacheStatusState::restored; } - - void CacheStatus::mark_unavailable(const IBinaryProvider* sender) - { - switch (m_status) - { - case CacheStatusState::unknown: - if (!Util::Vectors::contains(m_known_unavailable_providers, sender)) - { - m_known_unavailable_providers.push_back(sender); - } - break; - case CacheStatusState::available: - case CacheStatusState::restored: break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - void CacheStatus::mark_available(const IBinaryProvider* sender) noexcept - { - switch (m_status) - { - case CacheStatusState::unknown: - m_known_unavailable_providers.~vector(); - m_status = CacheStatusState::available; - m_available_provider = sender; - break; - case CacheStatusState::available: - case CacheStatusState::restored: break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - void CacheStatus::mark_restored() noexcept - { - switch (m_status) - { - case CacheStatusState::unknown: m_known_unavailable_providers.~vector(); [[fallthrough]]; - case CacheStatusState::available: m_status = CacheStatusState::restored; break; - case CacheStatusState::restored: break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - const IBinaryProvider* CacheStatus::get_available_provider() const noexcept - { - switch (m_status) - { - case CacheStatusState::available: return m_available_provider; - case CacheStatusState::unknown: - case CacheStatusState::restored: return nullptr; - default: Checks::unreachable(VCPKG_LINE_INFO); + return flatten(cmd_execute_and_capture_output(cmd), Tools::COSCLI); } - } - void BinaryConfigParserState::clear() - { - binary_cache_providers.clear(); - binary_cache_providers.insert("clear"); - nuget_interactive = false; - nugettimeout = "100"; - archives_to_read.clear(); - archives_to_write.clear(); - url_templates_to_get.clear(); - url_templates_to_put.clear(); - gcs_read_prefixes.clear(); - gcs_write_prefixes.clear(); - aws_read_prefixes.clear(); - aws_write_prefixes.clear(); - aws_no_sign_request = false; - cos_read_prefixes.clear(); - cos_write_prefixes.clear(); - gha_read = false; - gha_write = false; - sources_to_read.clear(); - sources_to_write.clear(); - configs_to_read.clear(); - configs_to_write.clear(); - secrets.clear(); - } + ExpectedL upload_file(StringView object, const Path& archive) const override + { + auto cmd = command().string_arg("cp").string_arg(archive).string_arg(object); + return flatten(cmd_execute_and_capture_output(cmd), Tools::COSCLI); + } - BinaryPackageInformation::BinaryPackageInformation(const InstallPlanAction& action, Optional nuspec) - : package_abi(action.package_abi().value_or_exit(VCPKG_LINE_INFO)) - , spec(action.spec) - , raw_version(action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO) - .source_control_file->core_paragraph->raw_version) - , nuspec(std::move(nuspec)) - { - } -} + Command m_tool; + }; -namespace -{ ExpectedL default_cache_path_impl() { auto maybe_cachepath = get_environment_variable("VCPKG_DEFAULT_BINARY_CACHE"); @@ -2026,14 +1364,8 @@ namespace return add_error(msg::format(msgNugetTimeoutExpectsSinglePositiveInteger)); } - auto&& t = segments[1].second; - if (t.empty()) - { - return add_error(msg::format(msgNugetTimeoutExpectsSinglePositiveInteger)); - } - char* end; - long timeout = std::strtol(t.c_str(), &end, 0); - if (*end != '\0' || timeout <= 0) + long timeout = Strings::strto(segments[1].second).value_or(-1); + if (timeout <= 0) { return add_error(msg::format(msgNugetTimeoutExpectsSinglePositiveInteger)); } @@ -2106,10 +1438,12 @@ namespace p.append(segments[2].second); state->secrets.push_back(segments[2].second); UrlTemplate url_template = {p}; + bool read = false, write = false; + handle_readwrite(read, write, segments, 3); + if (read) state->url_templates_to_get.push_back(url_template); auto headers = azure_blob_headers(); - url_template.headers_for_put.assign(headers.begin(), headers.end()); - handle_readwrite( - state->url_templates_to_get, state->url_templates_to_put, std::move(url_template), segments, 3); + url_template.headers.assign(headers.begin(), headers.end()); + if (write) state->url_templates_to_put.push_back(url_template); state->binary_cache_providers.insert("azblob"); } @@ -2281,8 +1615,7 @@ namespace } if (segments.size() == 4) { - url_template.headers_for_get.push_back(segments[3].second); - url_template.headers_for_put.push_back(segments[3].second); + url_template.headers.push_back(segments[3].second); } handle_readwrite( @@ -2293,6 +1626,219 @@ namespace { return add_error(msg::format(msgUnknownBinaryProviderType), segments[0].first); } + } + }; + + struct AssetSourcesState + { + bool cleared = false; + bool block_origin = false; + std::vector url_templates_to_get; + std::vector azblob_templates_to_put; + std::vector secrets; + Optional script; + + void clear() + { + cleared = true; + block_origin = false; + url_templates_to_get.clear(); + azblob_templates_to_put.clear(); + secrets.clear(); + script = nullopt; + } + }; + + struct AssetSourcesParser : ConfigSegmentsParser + { + AssetSourcesParser(StringView text, StringView origin, AssetSourcesState* state) + : ConfigSegmentsParser(text, origin), state(state) + { + } + + AssetSourcesState* state; + + void parse() + { + auto all_segments = parse_all_segments(); + for (auto&& x : all_segments) + { + if (get_error()) return; + handle_segments(std::move(x)); + } + } + + void handle_segments(std::vector>&& segments) + { + Checks::check_exit(VCPKG_LINE_INFO, !segments.empty()); + + if (segments[0].second == "x-block-origin") + { + if (segments.size() >= 2) + { + return add_error( + msg::format(msgAssetCacheProviderAcceptsNoArguments, msg::value = "x-block-origin"), + segments[1].first); + } + + state->block_origin = true; + } + else if (segments[0].second == "clear") + { + if (segments.size() >= 2) + { + return add_error(msg::format(msgAssetCacheProviderAcceptsNoArguments, msg::value = "clear"), + segments[1].first); + } + + state->clear(); + } + else if (segments[0].second == "x-azurl") + { + // Scheme: x-azurl,[,[,]] + if (segments.size() < 2) + { + return add_error(msg::format(msgAzUrlAssetCacheRequiresBaseUrl), segments[0].first); + } + + if (segments.size() > 4) + { + return add_error(msg::format(msgAzUrlAssetCacheRequiresLessThanFour), segments[4].first); + } + + if (segments[1].second.empty()) + { + return add_error(msg::format(msgAzUrlAssetCacheRequiresBaseUrl), segments[1].first); + } + + auto p = segments[1].second; + if (p.back() != '/') + { + p.push_back('/'); + } + + p.append(""); + if (segments.size() > 2 && !segments[2].second.empty()) + { + if (!Strings::starts_with(segments[2].second, "?")) + { + p.push_back('?'); + } + p.append(segments[2].second); + // Note: the download manager does not currently respect secrets + state->secrets.push_back(segments[2].second); + } + handle_readwrite( + state->url_templates_to_get, state->azblob_templates_to_put, std::move(p), segments, 3); + } + else if (segments[0].second == "x-script") + { + // Scheme: x-script, + if (segments.size() != 2) + { + return add_error(msg::format(msgScriptAssetCacheRequiresScript), segments[0].first); + } + state->script = segments[1].second; + } + else + { + // Don't forget to update this message if new providers are added. + return add_error(msg::format(msgUnexpectedAssetCacheProvider), segments[0].first); + } + } + }; +} + +namespace vcpkg +{ + LocalizedString UrlTemplate::valid() const + { + std::vector invalid_keys; + auto result = api_stable_format(url_template, [&](std::string&, StringView key) { + static constexpr std::array valid_keys = {"name", "version", "sha", "triplet"}; + if (!Util::Vectors::contains(valid_keys, key)) + { + invalid_keys.push_back(key.to_string()); + } + }); + if (!result) + { + return result.error(); + } + if (!invalid_keys.empty()) + { + return msg::format(msgUnknownVariablesInTemplate, + msg::value = url_template, + msg::list = Strings::join(", ", invalid_keys)); + } + return {}; + } + + std::string UrlTemplate::instantiate_variables(const BinaryPackageReadInfo& info) const + { + return api_stable_format(url_template, + [&](std::string& out, StringView key) { + if (key == "version") + { + out += info.raw_version; + } + else if (key == "name") + { + out += info.spec.name(); + } + else if (key == "triplet") + { + out += info.spec.triplet().canonical_name(); + } + else if (key == "sha") + { + out += info.package_abi; + } + else + { + Debug::println("Unknown key: ", key); + // We do a input validation while parsing the config + Checks::unreachable(VCPKG_LINE_INFO); + }; + }) + .value_or_exit(VCPKG_LINE_INFO); + } + + ExpectedL make_binary_providers(const VcpkgCmdArguments& args, const VcpkgPaths& paths) + { + BinaryProviders ret; + if (args.binary_caching_enabled()) + { + if (Debug::g_debugging) + { + const auto& maybe_cachepath = default_cache_path(); + if (const auto cachepath = maybe_cachepath.get()) + { + Debug::print("Default binary cache path is: ", *cachepath, '\n'); + } + else + { + Debug::print("No binary cache path. Reason: ", maybe_cachepath.error(), '\n'); + } + } + + if (args.env_binary_sources.has_value()) + { + get_global_metrics_collector().track_define(DefineMetric::VcpkgBinarySources); + } + + if (args.cli_binary_sources.size() != 0) + { + get_global_metrics_collector().track_define(DefineMetric::BinaryCachingSource); + } + + auto sRawHolder = + parse_binary_provider_configs(args.env_binary_sources.value_or(""), args.cli_binary_sources); + if (!sRawHolder) + { + return std::move(sRawHolder).error(); + } + auto& s = *sRawHolder.get(); static const std::map metric_names{ {"aws", DefineMetric::BinaryCachingAws}, @@ -2306,7 +1852,7 @@ namespace }; MetricsSubmission metrics; - for (const auto& cache_provider : state->binary_cache_providers) + for (const auto& cache_provider : s.binary_cache_providers) { auto it = metric_names.find(cache_provider); if (it != metric_names.end()) @@ -2316,127 +1862,422 @@ namespace } get_global_metrics_collector().track_submission(std::move(metrics)); - } - }; - struct AssetSourcesState + s.nuget_prefix = args.nuget_id_prefix.value_or(""); + if (!s.nuget_prefix.empty()) s.nuget_prefix.push_back('_'); + s.use_nuget_cache = args.use_nuget_cache.value_or(false); + s.nuget_repo_info = get_nuget_repo_info_from_env(); + + auto& fs = paths.get_filesystem(); + auto& tools = paths.get_tool_cache(); + const auto& buildtrees = paths.buildtrees(); + + ret.nuget_prefix = s.nuget_prefix; + + Optional lazy_zip_tool; + auto zip_tool = [&]() -> const ZipTool& { + if (!lazy_zip_tool) lazy_zip_tool.emplace(fs, tools, stdout_sink); + return *lazy_zip_tool.get(); + }; + + std::shared_ptr lazy_gcs_tool; + auto gcs_tool = [&]() -> const std::shared_ptr& { + if (!lazy_gcs_tool) lazy_gcs_tool = std::make_shared(tools, stdout_sink); + return lazy_gcs_tool; + }; + + std::shared_ptr lazy_aws_tool; + auto aws_tool = [&]() -> const std::shared_ptr& { + if (!lazy_aws_tool) + lazy_aws_tool = std::make_shared(tools, stdout_sink, s.aws_no_sign_request); + return lazy_aws_tool; + }; + + std::shared_ptr lazy_cos_tool; + auto cos_tool = [&]() -> const std::shared_ptr& { + if (!lazy_cos_tool) lazy_cos_tool = std::make_shared(tools, stdout_sink); + return lazy_cos_tool; + }; + Optional lazy_nuget_base; + auto nuget_base = [&]() -> const NugetBaseBinaryProvider& { + if (!lazy_nuget_base) + lazy_nuget_base.emplace( + fs, NuGetTool(tools, stdout_sink, s), paths.packages(), buildtrees, s.nuget_prefix); + return *lazy_nuget_base.get(); + }; + + for (auto&& dir : s.archives_to_read) + ret.read.push_back(std::make_unique(zip_tool(), fs, std::move(dir))); + if (!s.archives_to_write.empty()) + ret.write.push_back(std::make_unique(fs, std::move(s.archives_to_write))); + + for (auto&& url : s.url_templates_to_get) + ret.read.push_back( + std::make_unique(zip_tool(), fs, buildtrees, std::move(url), s.secrets)); + if (!s.url_templates_to_put.empty()) + ret.write.push_back( + std::make_unique(fs, std::move(s.url_templates_to_put), s.secrets)); + + for (auto&& prefix : s.gcs_read_prefixes) + ret.read.push_back( + std::make_unique(zip_tool(), fs, buildtrees, std::move(prefix), gcs_tool())); + if (!s.gcs_write_prefixes.empty()) + ret.write.push_back( + std::make_unique(std::move(s.gcs_write_prefixes), gcs_tool())); + + for (auto&& prefix : s.aws_read_prefixes) + ret.read.push_back( + std::make_unique(zip_tool(), fs, buildtrees, std::move(prefix), aws_tool())); + if (!s.aws_write_prefixes.empty()) + ret.write.push_back( + std::make_unique(std::move(s.aws_write_prefixes), aws_tool())); + + for (auto&& prefix : s.cos_read_prefixes) + ret.read.push_back( + std::make_unique(zip_tool(), fs, buildtrees, std::move(prefix), cos_tool())); + if (!s.cos_write_prefixes.empty()) + ret.write.push_back( + std::make_unique(std::move(s.cos_write_prefixes), cos_tool())); + + if (s.gha_read || s.gha_write) + { + if (!args.actions_cache_url.has_value() || !args.actions_runtime_token.has_value()) + return msg::format_error(msgGHAParametersMissing, + msg::url = + "https://learn.microsoft.com/en-us/vcpkg/users/binarycaching#gha"); + const auto& url = *args.actions_cache_url.get(); + const auto& token = *args.actions_runtime_token.get(); + if (s.gha_read) + ret.read.push_back(std::make_unique(zip_tool(), fs, buildtrees, url, token)); + if (s.gha_write) ret.write.push_back(std::make_unique(fs, url, token)); + } + + if (!s.sources_to_read.empty()) + ret.read.push_back( + std::make_unique(nuget_base(), nuget_sources_arg(s.sources_to_read))); + for (auto&& config : s.configs_to_read) + ret.read.push_back( + std::make_unique(nuget_base(), nuget_configfile_arg(config))); + if (!s.sources_to_write.empty() || !s.configs_to_write.empty()) + { + ret.write.push_back(std::make_unique( + nuget_base(), std::move(s.sources_to_write), std::move(s.configs_to_write))); + } + } + return std::move(ret); + } + + ReadOnlyBinaryCache::ReadOnlyBinaryCache(const VcpkgCmdArguments& args, + const VcpkgPaths& paths, + const LineInfo& info) + : ReadOnlyBinaryCache(make_binary_providers(args, paths).value_or_exit(info)) { - bool cleared = false; - bool block_origin = false; - std::vector url_templates_to_get; - std::vector azblob_templates_to_put; - std::vector secrets; - Optional script; + } - void clear() + ReadOnlyBinaryCache::ReadOnlyBinaryCache(BinaryProviders&& providers) : m_config(std::move(providers)) { } + + RestoreResult ReadOnlyBinaryCache::try_restore(const InstallPlanAction& action) + { + const auto abi = action.package_abi().get(); + if (!abi) { - cleared = true; - block_origin = false; - url_templates_to_get.clear(); - azblob_templates_to_put.clear(); - secrets.clear(); - script = nullopt; + // e.g. this is a `--head` package + return RestoreResult::unavailable; } - }; - struct AssetSourcesParser : ConfigSegmentsParser - { - AssetSourcesParser(StringView text, StringView origin, AssetSourcesState* state) - : ConfigSegmentsParser(text, origin), state(state) + auto& cache_status = m_status[*abi]; + if (cache_status.is_restored()) { + return RestoreResult::restored; } - AssetSourcesState* state; - - void parse() + const auto available = cache_status.get_available_provider(); + if (available) { - auto all_segments = parse_all_segments(); - for (auto&& x : all_segments) + switch (available->try_restore(action)) { - if (get_error()) return; - handle_segments(std::move(x)); + case RestoreResult::unavailable: + // Even though that provider thought it had it, it didn't; perhaps + // due to intermittent network problems etc. + // Try other providers below + break; + case RestoreResult::restored: cache_status.mark_restored(); return RestoreResult::restored; + default: Checks::unreachable(VCPKG_LINE_INFO); } } - void handle_segments(std::vector>&& segments) + for (auto&& provider : m_config.read) { - Checks::check_exit(VCPKG_LINE_INFO, !segments.empty()); + if (provider.get() == available || cache_status.is_unavailable(provider.get())) + { + continue; // this one was already tried or knows the package is unavailable + } - if (segments[0].second == "x-block-origin") + switch (provider->try_restore(action)) { - if (segments.size() >= 2) + case RestoreResult::restored: cache_status.mark_restored(); return RestoreResult::restored; + case RestoreResult::unavailable: cache_status.mark_unavailable(provider.get()); break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + return RestoreResult::unavailable; + } + + void ReadOnlyBinaryCache::prefetch(View actions) + { + std::vector action_ptrs; + std::vector restores; + std::vector statuses; + for (auto&& provider : m_config.read) + { + action_ptrs.clear(); + restores.clear(); + statuses.clear(); + for (size_t i = 0; i < actions.size(); ++i) + { + if (actions[i].package_abi()) { - return add_error( - msg::format(msgAssetCacheProviderAcceptsNoArguments, msg::value = "x-block-origin"), - segments[1].first); + CacheStatus* status = &m_status[*actions[i].package_abi().get()]; + if (status->should_attempt_restore(provider.get())) + { + action_ptrs.push_back(&actions[i]); + restores.push_back(RestoreResult::unavailable); + statuses.push_back(status); + } } - - state->block_origin = true; } - else if (segments[0].second == "clear") + if (action_ptrs.empty()) continue; + + ElapsedTimer timer; + provider->prefetch(action_ptrs, restores); + size_t num_restored = 0; + for (size_t i = 0; i < restores.size(); ++i) { - if (segments.size() >= 2) + if (restores[i] == RestoreResult::unavailable) { - return add_error(msg::format(msgAssetCacheProviderAcceptsNoArguments, msg::value = "clear"), - segments[1].first); + statuses[i]->mark_unavailable(provider.get()); + } + else + { + statuses[i]->mark_restored(); + ++num_restored; } - - state->clear(); } - else if (segments[0].second == "x-azurl") + msg::println(msgRestoredPackagesFromVendor, + msg::count = num_restored, + msg::elapsed = timer.elapsed(), + msg::value = provider->vendor()); + } + } + + std::vector ReadOnlyBinaryCache::precheck(View actions) + { + std::vector statuses = Util::fmap(actions, [this](const auto& action) { + if (!action.package_abi()) Checks::unreachable(VCPKG_LINE_INFO); + return &m_status[*action.package_abi().get()]; + }); + + std::vector action_ptrs; + std::vector cache_result; + std::vector indexes; + for (auto&& provider : m_config.read) + { + action_ptrs.clear(); + cache_result.clear(); + indexes.clear(); + for (size_t i = 0; i < actions.size(); ++i) { - // Scheme: x-azurl,[,[,]] - if (segments.size() < 2) + if (statuses[i]->should_attempt_precheck(provider.get())) { - return add_error(msg::format(msgAzUrlAssetCacheRequiresBaseUrl), segments[0].first); + action_ptrs.push_back(&actions[i]); + cache_result.push_back(CacheAvailability::unknown); + indexes.push_back(i); } + } + if (action_ptrs.empty()) continue; - if (segments.size() > 4) + provider->precheck(action_ptrs, cache_result); + + for (size_t i = 0; i < action_ptrs.size(); ++i) + { + if (cache_result[i] == CacheAvailability::available) { - return add_error(msg::format(msgAzUrlAssetCacheRequiresLessThanFour), segments[4].first); + m_status[*action_ptrs[i]->package_abi().get()].mark_available(provider.get()); } - - if (segments[1].second.empty()) + else if (cache_result[i] == CacheAvailability::unavailable) { - return add_error(msg::format(msgAzUrlAssetCacheRequiresBaseUrl), segments[1].first); + m_status[*action_ptrs[i]->package_abi().get()].mark_unavailable(provider.get()); } + } + } - auto p = segments[1].second; - if (p.back() != '/') + std::vector ret = Util::fmap(statuses, [](CacheStatus* s) { + return s->get_available_provider() ? CacheAvailability::available : CacheAvailability::unavailable; + }); + return ret; + } + + BinaryCache::BinaryCache(RemoveFilesystem& fs) : m_fs(fs) { } + + BinaryCache::BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info) + : BinaryCache( + make_binary_providers(args, paths).value_or_exit(info), paths.get_filesystem(), paths.get_tool_cache()) + { + } + + BinaryCache::BinaryCache(BinaryProviders&& providers, RemoveFilesystem& fs, const ToolCache& tools) + : ReadOnlyBinaryCache(std::move(providers)), m_fs(fs) + { + m_needs_nuspec_data = Util::any_of(m_config.write, [](auto&& p) { return p->needs_nuspec_data(); }); + m_needs_zip_file = Util::any_of(m_config.write, [](auto&& p) { return p->needs_zip_file(); }); + if (m_needs_zip_file) m_zip_tool.emplace(fs, tools, stdout_sink); + } + BinaryCache::~BinaryCache() { } + + void BinaryCache::push_success(const InstallPlanAction& action) + { + if (action.package_abi().has_value()) + { + bool restored = m_status[*action.package_abi().get()].is_restored(); + // Purge all status information on push_success (cache invalidation) + // - push_success may delete packages/ (invalidate restore) + // - push_success may make the package available from providers (invalidate valid) + m_status.erase(*action.package_abi().get()); + if (!restored && !m_config.write.empty()) + { + ElapsedTimer timer; + BinaryPackageWriteInfo request{action}; + + if (m_needs_nuspec_data) { - p.push_back('/'); + request.nuspec = + generate_nuspec(request.package_dir, action, m_config.nuget_prefix, m_config.nuget_repo); } - - p.append(""); - if (segments.size() > 2 && !segments[2].second.empty()) + if (m_needs_zip_file) { - if (!Strings::starts_with(segments[2].second, "?")) + Path zip_path = request.package_dir + ".zip"; + auto compress_result = m_zip_tool.value_or_exit(VCPKG_LINE_INFO) + .compress_directory_to_zip(request.package_dir, zip_path); + if (!compress_result) { - p.push_back('?'); + stdout_sink.println( + Color::warning, + msg::format_warning(msgCompressFolderFailed, msg::path = request.package_dir) + .append_raw(' ') + .append_raw(compress_result.error())); + } + else + { + request.zip_path = std::move(zip_path); } - p.append(segments[2].second); - // Note: the download manager does not currently respect secrets - state->secrets.push_back(segments[2].second); } - handle_readwrite( - state->url_templates_to_get, state->azblob_templates_to_put, std::move(p), segments, 3); - } - else if (segments[0].second == "x-script") - { - // Scheme: x-script, - if (segments.size() != 2) + + size_t num_destinations = 0; + for (auto&& provider : m_config.write) { - return add_error(msg::format(msgScriptAssetCacheRequiresScript), segments[0].first); + num_destinations += provider->push_success(request, stdout_sink); } - state->script = segments[1].second; - } - else - { - // Don't forget to update this message if new providers are added. - return add_error(msg::format(msgUnexpectedAssetCacheProvider), segments[0].first); + if (request.zip_path) + { + m_fs.remove(*request.zip_path.get(), IgnoreErrors{}); + } + stdout_sink.println( + msgStoredBinariesToDestinations, msg::count = num_destinations, msg::elapsed = timer.elapsed()); } } - }; + if (action.build_options.clean_packages == CleanPackages::YES) + { + m_fs.remove_all(action.package_dir.value_or_exit(VCPKG_LINE_INFO), VCPKG_LINE_INFO); + } + } + + bool CacheStatus::should_attempt_precheck(const IReadBinaryProvider* sender) const noexcept + { + switch (m_status) + { + case CacheStatusState::unknown: return !Util::Vectors::contains(m_known_unavailable_providers, sender); + case CacheStatusState::available: return false; + case CacheStatusState::restored: return false; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + bool CacheStatus::should_attempt_restore(const IReadBinaryProvider* sender) const noexcept + { + switch (m_status) + { + case CacheStatusState::unknown: return !Util::Vectors::contains(m_known_unavailable_providers, sender); + case CacheStatusState::available: return m_available_provider == sender; + case CacheStatusState::restored: return false; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + bool CacheStatus::is_unavailable(const IReadBinaryProvider* sender) const noexcept + { + return Util::Vectors::contains(m_known_unavailable_providers, sender); + } + + bool CacheStatus::is_restored() const noexcept { return m_status == CacheStatusState::restored; } + + void CacheStatus::mark_unavailable(const IReadBinaryProvider* sender) + { + if (!Util::Vectors::contains(m_known_unavailable_providers, sender)) + { + m_known_unavailable_providers.push_back(sender); + } + } + void CacheStatus::mark_available(const IReadBinaryProvider* sender) noexcept + { + switch (m_status) + { + case CacheStatusState::unknown: + m_status = CacheStatusState::available; + m_available_provider = sender; + break; + case CacheStatusState::available: + case CacheStatusState::restored: break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + void CacheStatus::mark_restored() noexcept + { + switch (m_status) + { + case CacheStatusState::unknown: m_known_unavailable_providers.clear(); [[fallthrough]]; + case CacheStatusState::available: m_status = CacheStatusState::restored; break; + case CacheStatusState::restored: break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + const IReadBinaryProvider* CacheStatus::get_available_provider() const noexcept + { + switch (m_status) + { + case CacheStatusState::available: return m_available_provider; + case CacheStatusState::unknown: + case CacheStatusState::restored: return nullptr; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + void BinaryConfigParserState::clear() + { + *this = BinaryConfigParserState(); + binary_cache_providers.insert("clear"); + } + + BinaryPackageReadInfo::BinaryPackageReadInfo(const InstallPlanAction& action) + : package_abi(action.package_abi().value_or_exit(VCPKG_LINE_INFO)) + , spec(action.spec) + , raw_version(action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO) + .source_control_file->core_paragraph->raw_version) + , package_dir(action.package_dir.value_or_exit(VCPKG_LINE_INFO)) + { + } } ExpectedL vcpkg::parse_download_configuration(const Optional& arg) @@ -2495,19 +2336,9 @@ ExpectedL vcpkg::parse_download_configuration(const Optio s.script}; } -ExpectedL vcpkg::create_binary_providers_from_configs_pure(const std::string& env_string, - View args) +ExpectedL vcpkg::parse_binary_provider_configs(const std::string& env_string, + View args) { - if (!env_string.empty()) - { - get_global_metrics_collector().track_define(DefineMetric::VcpkgBinarySources); - } - - if (args.size() != 0) - { - get_global_metrics_collector().track_define(DefineMetric::BinaryCachingSource); - } - BinaryConfigParserState s; BinaryConfigParser default_parser("default,readwrite", "", &s); @@ -2537,90 +2368,6 @@ ExpectedL vcpkg::create_binary_providers_from_configs_p return s; } -ExpectedL>> vcpkg::create_binary_providers_from_configs( - const VcpkgPaths& paths, View args) -{ - std::string env_string = get_environment_variable("VCPKG_BINARY_SOURCES").value_or(""); - if (Debug::g_debugging) - { - const auto& maybe_cachepath = default_cache_path(); - if (const auto cachepath = maybe_cachepath.get()) - { - Debug::print("Default binary cache path is: ", *cachepath, '\n'); - } - else - { - Debug::print("No binary cache path. Reason: ", maybe_cachepath.error(), '\n'); - } - } - - auto sRawHolder = create_binary_providers_from_configs_pure(env_string, args); - if (!sRawHolder) - { - return sRawHolder.error(); - } - - auto& s = sRawHolder.value_or_exit(VCPKG_LINE_INFO); - std::vector> providers; - if (!s.archives_to_read.empty() || !s.archives_to_write.empty() || !s.url_templates_to_put.empty()) - { - providers.push_back(std::make_unique(paths, - std::move(s.archives_to_read), - std::move(s.archives_to_write), - std::move(s.url_templates_to_put), - s.secrets)); - } - - if (!s.gcs_read_prefixes.empty() || !s.gcs_write_prefixes.empty()) - { - providers.push_back(std::make_unique( - paths, std::move(s.gcs_read_prefixes), std::move(s.gcs_write_prefixes))); - } - - if (!s.aws_read_prefixes.empty() || !s.aws_write_prefixes.empty()) - { - providers.push_back(std::make_unique( - paths, std::move(s.aws_read_prefixes), std::move(s.aws_write_prefixes), s.aws_no_sign_request)); - } - - if (!s.cos_read_prefixes.empty() || !s.cos_write_prefixes.empty()) - { - providers.push_back(std::make_unique( - paths, std::move(s.cos_read_prefixes), std::move(s.cos_write_prefixes))); - } - - if (s.gha_read || s.gha_write) - { - auto url = get_environment_variable("ACTIONS_CACHE_URL"); - auto token = get_environment_variable("ACTIONS_RUNTIME_TOKEN"); - Checks::msg_check_exit(VCPKG_LINE_INFO, - (url.has_value() && token.has_value()), - msgGHAParametersMissing, - msg::url = "https://learn.microsoft.com/en-us/vcpkg/users/binarycaching#gha"); - providers.push_back(std::make_unique(paths, s.gha_read, s.gha_write, url, token)); - } - - if (!s.url_templates_to_get.empty()) - { - providers.push_back( - std::make_unique(paths, std::move(s.url_templates_to_get), s.secrets)); - } - - if (!s.sources_to_read.empty() || !s.sources_to_write.empty() || !s.configs_to_read.empty() || - !s.configs_to_write.empty()) - { - providers.push_back(std::make_unique(paths, - std::move(s.sources_to_read), - std::move(s.sources_to_write), - std::move(s.configs_to_read), - std::move(s.configs_to_write), - std::move(s.nugettimeout), - s.nuget_interactive)); - } - - return providers; -} - std::string vcpkg::format_version_for_nugetref(StringView version, StringView abi_tag) { // this cannot use DotVersion::try_parse or DateVersion::try_parse, @@ -2649,7 +2396,7 @@ std::string vcpkg::format_version_for_nugetref(StringView version, StringView ab return Strings::concat("0.0.0-vcpkg", abi_tag); } -details::NuGetRepoInfo details::get_nuget_repo_info_from_env() +NuGetRepoInfo vcpkg::get_nuget_repo_info_from_env() { auto vcpkg_nuget_repository = get_environment_variable("VCPKG_NUGET_REPOSITORY"); if (auto p = vcpkg_nuget_repository.get()) @@ -2678,14 +2425,15 @@ details::NuGetRepoInfo details::get_nuget_repo_info_from_env() std::string vcpkg::generate_nuspec(const Path& package_dir, const InstallPlanAction& action, - const vcpkg::NugetReference& ref, - details::NuGetRepoInfo rinfo) + StringView id_prefix, + NuGetRepoInfo rinfo) { auto& spec = action.spec; auto& scf = *action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO).source_control_file; auto& version = scf.core_paragraph->raw_version; const auto& abi_info = action.abi_info.value_or_exit(VCPKG_LINE_INFO); const auto& compiler_info = abi_info.compiler_info.value_or_exit(VCPKG_LINE_INFO); + auto ref = make_nugetref(action, id_prefix); std::string description = Strings::concat("NOT FOR DIRECT USE. Automatically generated cache package.\n\n", Strings::join("\n ", scf.core_paragraph->description), @@ -2815,15 +2563,14 @@ LocalizedString vcpkg::format_help_topic_binary_caching() .append(msgExtendedDocumentationAtUrl, msg::url = docs::binarycaching_url); } -std::string vcpkg::generate_nuget_packages_config(const ActionPlan& action) +std::string vcpkg::generate_nuget_packages_config(const ActionPlan& plan, StringView prefix) { - auto refs = Util::fmap(action.install_actions, - [&](const InstallPlanAction& ipa) { return make_nugetref(ipa, get_nuget_prefix()); }); XmlSerializer xml; xml.emit_declaration().line_break(); xml.open_tag("packages").line_break(); - for (auto&& ref : refs) + for (auto&& action : plan.install_actions) { + auto ref = make_nugetref(action, prefix); xml.start_complex_open_tag("package") .text_attr("id", ref.id) .text_attr("version", ref.version) @@ -2834,3 +2581,12 @@ std::string vcpkg::generate_nuget_packages_config(const ActionPlan& action) xml.close_tag("packages").line_break(); return std::move(xml.buf); } + +NugetReference vcpkg::make_nugetref(const InstallPlanAction& action, StringView prefix) +{ + return ::make_nugetref(action.spec, + action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO) + .source_control_file->core_paragraph->raw_version, + action.abi_info.value_or_exit(VCPKG_LINE_INFO).package_abi, + prefix); +} diff --git a/src/vcpkg/commands.build-external.cpp b/src/vcpkg/commands.build-external.cpp index 907ea18dbc..cf8819adbb 100644 --- a/src/vcpkg/commands.build-external.cpp +++ b/src/vcpkg/commands.build-external.cpp @@ -25,8 +25,6 @@ namespace vcpkg::Commands::BuildExternal { const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); - BinaryCache binary_cache{args, paths}; - bool default_triplet_used = false; const FullPackageSpec spec = check_and_get_full_package_spec(options.command_arguments[0], default_triplet, @@ -44,6 +42,6 @@ namespace vcpkg::Commands::BuildExternal auto& fs = paths.get_filesystem(); auto registry_set = paths.make_registry_set(); PathsPortFileProvider provider(fs, *registry_set, make_overlay_provider(fs, paths.original_cwd, overlays)); - Build::perform_and_exit_ex(args, spec, host_triplet, provider, binary_cache, null_build_logs_recorder(), paths); + Build::perform_and_exit_ex(args, spec, host_triplet, provider, null_build_logs_recorder(), paths); } } diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 92cf5ad08f..14f1fc0945 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -60,13 +60,11 @@ namespace vcpkg::Build const FullPackageSpec& full_spec, Triplet host_triplet, const PathsPortFileProvider& provider, - BinaryCache& binary_cache, const IBuildLogsRecorder& build_logs_recorder, const VcpkgPaths& paths) { - Checks::exit_with_code( - VCPKG_LINE_INFO, - perform_ex(args, full_spec, host_triplet, provider, binary_cache, build_logs_recorder, paths)); + Checks::exit_with_code(VCPKG_LINE_INFO, + perform_ex(args, full_spec, host_triplet, provider, build_logs_recorder, paths)); } const CommandStructure COMMAND_STRUCTURE = { @@ -89,7 +87,6 @@ namespace vcpkg::Build const FullPackageSpec& full_spec, Triplet host_triplet, const PathsPortFileProvider& provider, - BinaryCache& binary_cache, const IBuildLogsRecorder& build_logs_recorder, const VcpkgPaths& paths) { @@ -99,8 +96,8 @@ namespace vcpkg::Build var_provider.load_dep_info_vars({{spec}}, host_triplet); StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); - auto action_plan = - create_feature_install_plan(provider, var_provider, {&full_spec, 1}, status_db, {host_triplet}); + auto action_plan = create_feature_install_plan( + provider, var_provider, {&full_spec, 1}, status_db, {host_triplet, paths.packages()}); var_provider.load_tag_vars(action_plan, provider, host_triplet); @@ -141,6 +138,7 @@ namespace vcpkg::Build action->build_options.clean_buildtrees = CleanBuildtrees::NO; action->build_options.clean_packages = CleanPackages::NO; + BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); const ElapsedTimer build_timer; const auto result = build_package(args, paths, *action, build_logs_recorder, status_db); msg::print(msgElapsedForPackage, msg::spec = spec, msg::elapsed = build_timer); @@ -172,7 +170,7 @@ namespace vcpkg::Build msg::print(create_user_troubleshooting_message(*action, paths, nullopt)); return 1; } - binary_cache.push_success(*action, paths.package_dir(action->spec)); + binary_cache.push_success(*action); return 0; } @@ -181,7 +179,6 @@ namespace vcpkg::Build { // Build only takes a single package and all dependencies must already be installed const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); - BinaryCache binary_cache{args, paths}; bool default_triplet_used = false; const FullPackageSpec spec = check_and_get_full_package_spec(options.command_arguments[0], default_triplet, @@ -197,7 +194,7 @@ namespace vcpkg::Build auto registry_set = paths.make_registry_set(); PathsPortFileProvider provider( fs, *registry_set, make_overlay_provider(fs, paths.original_cwd, paths.overlay_ports)); - return perform_ex(args, spec, host_triplet, provider, binary_cache, null_build_logs_recorder(), paths); + return perform_ex(args, spec, host_triplet, provider, null_build_logs_recorder(), paths); } } // namespace vcpkg::Build diff --git a/src/vcpkg/commands.check-support.cpp b/src/vcpkg/commands.check-support.cpp index bb21805098..a3a20a6938 100644 --- a/src/vcpkg/commands.check-support.cpp +++ b/src/vcpkg/commands.check-support.cpp @@ -131,9 +131,10 @@ namespace vcpkg::Commands auto cmake_vars = CMakeVars::make_triplet_cmake_var_provider(paths); // for each spec in the user-requested specs, check all dependencies + CreateInstallPlanOptions create_options{host_triplet, paths.packages()}; for (const auto& user_spec : specs) { - auto action_plan = create_feature_install_plan(provider, *cmake_vars, {&user_spec, 1}, {}, {host_triplet}); + auto action_plan = create_feature_install_plan(provider, *cmake_vars, {&user_spec, 1}, {}, create_options); cmake_vars->load_tag_vars(action_plan, provider, host_triplet); diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 29f265099a..dacc26d684 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -351,8 +351,6 @@ namespace vcpkg::Commands::CI const ParsedArguments options = args.parse_arguments(COMMAND_STRUCTURE); const auto& settings = options.settings; - BinaryCache binary_cache{args, paths}; - ExclusionsMap exclusions_map; parse_exclusions(settings, OPTION_EXCLUDE, target_triplet, exclusions_map); parse_exclusions(settings, OPTION_HOST_EXCLUDE, host_triplet, exclusions_map); @@ -413,7 +411,7 @@ namespace vcpkg::Commands::CI InternalFeatureSet{"core", "default"}); } - CreateInstallPlanOptions serialize_options(host_triplet, UnsupportedPortAction::Warn); + CreateInstallPlanOptions serialize_options(host_triplet, paths.packages(), UnsupportedPortAction::Warn); struct RandomizerInstance : GraphRandomizer { @@ -433,6 +431,7 @@ namespace vcpkg::Commands::CI } auto action_plan = compute_full_plan(paths, provider, var_provider, all_default_full_specs, serialize_options); + BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); const auto precheck_results = binary_cache.precheck(action_plan.install_actions); auto split_specs = compute_action_statuses(ExclusionPredicate{&exclusions_map}, var_provider, precheck_results, action_plan); @@ -508,7 +507,7 @@ namespace vcpkg::Commands::CI { StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); auto summary = Install::perform( - args, action_plan, KeepGoing::YES, paths, status_db, binary_cache, build_logs_recorder, var_provider); + args, action_plan, KeepGoing::YES, paths, status_db, build_logs_recorder, var_provider); for (auto&& result : summary.results) { diff --git a/src/vcpkg/commands.dependinfo.cpp b/src/vcpkg/commands.dependinfo.cpp index 75cbaa00ce..4584b3dd08 100644 --- a/src/vcpkg/commands.dependinfo.cpp +++ b/src/vcpkg/commands.dependinfo.cpp @@ -320,7 +320,7 @@ namespace vcpkg::Commands::DependInfo // All actions in the plan should be install actions, as there's no installed packages to remove. StatusParagraphs status_db; auto action_plan = create_feature_install_plan( - provider, var_provider, specs, status_db, {host_triplet, UnsupportedPortAction::Warn}); + provider, var_provider, specs, status_db, {host_triplet, paths.packages(), UnsupportedPortAction::Warn}); action_plan.print_unsupported_warnings(); if (!action_plan.remove_actions.empty()) diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index e3b4555e3f..d56c7ea37b 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -381,14 +381,7 @@ namespace vcpkg case InstallResult::FILE_CONFLICTS: code = BuildResult::FILE_CONFLICTS; break; default: Checks::unreachable(VCPKG_LINE_INFO); } - if (restore != RestoreResult::restored) - { - binary_cache.push_success(action, paths.package_dir(action.spec)); - } - else if (action.build_options.clean_packages == CleanPackages::YES) - { - fs.remove_all(paths.package_dir(action.spec), VCPKG_LINE_INFO); - } + binary_cache.push_success(action); if (action.build_options.clean_downloads == CleanDownloads::YES) { @@ -526,7 +519,6 @@ namespace vcpkg const KeepGoing keep_going, const VcpkgPaths& paths, StatusParagraphs& status_db, - BinaryCache& binary_cache, const IBuildLogsRecorder& build_logs_recorder, const CMakeVars::CMakeVarProvider& var_provider) { @@ -542,6 +534,7 @@ namespace vcpkg results.back().build_result.emplace(BuildResult::REMOVED); } + BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); for (auto&& action : action_plan.already_installed) { results.emplace_back(action); @@ -1028,12 +1021,6 @@ namespace vcpkg } } - BinaryCache binary_cache(paths.get_filesystem()); - if (!only_downloads) - { - binary_cache.install_providers_for(args, paths); - } - auto& fs = paths.get_filesystem(); DownloadTool download_tool = DownloadTool::BUILT_IN; @@ -1056,6 +1043,8 @@ namespace vcpkg auto var_provider_storage = CMakeVars::make_triplet_cmake_var_provider(paths); auto& var_provider = *var_provider_storage; + const CreateInstallPlanOptions create_options{host_triplet, paths.packages(), unsupported_port_action}; + if (auto manifest = paths.get_manifest().get()) { Optional pkgsconfig; @@ -1159,8 +1148,7 @@ namespace vcpkg dependencies, manifest_core.overrides, toplevel, - host_triplet, - unsupported_port_action) + create_options) .value_or_exit(VCPKG_LINE_INFO); install_plan.print_unsupported_warnings(); @@ -1179,7 +1167,6 @@ namespace vcpkg Commands::SetInstalled::perform_and_exit_ex(args, paths, provider, - binary_cache, var_provider, std::move(install_plan), dry_run ? Commands::SetInstalled::DryRun::Yes @@ -1211,8 +1198,7 @@ namespace vcpkg StatusParagraphs status_db = database_load_check(fs, paths.installed()); // Note: action_plan will hold raw pointers to SourceControlFileLocations from this map - auto action_plan = create_feature_install_plan( - provider, var_provider, specs, status_db, {host_triplet, unsupported_port_action}); + auto action_plan = create_feature_install_plan(provider, var_provider, specs, status_db, create_options); action_plan.print_unsupported_warnings(); for (auto&& action : action_plan.install_actions) @@ -1268,7 +1254,7 @@ namespace vcpkg compute_all_abis(paths, action_plan, var_provider, status_db); auto pkgsconfig_path = paths.original_cwd / it_pkgsconfig->second; - auto pkgsconfig_contents = generate_nuget_packages_config(action_plan); + auto pkgsconfig_contents = generate_nuget_packages_config(action_plan, args.nuget_id_prefix.value_or("")); fs.write_contents(pkgsconfig_path, pkgsconfig_contents, VCPKG_LINE_INFO); msg::println(msgWroteNuGetPkgConfInfo, msg::path = pkgsconfig_path); } @@ -1282,8 +1268,8 @@ namespace vcpkg track_install_plan(action_plan); - const InstallSummary summary = Install::perform( - args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder(), var_provider); + const InstallSummary summary = + Install::perform(args, action_plan, keep_going, paths, status_db, null_build_logs_recorder(), var_provider); if (keep_going == KeepGoing::YES) { diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 27134a22e1..d89bfaa983 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -45,7 +45,6 @@ namespace vcpkg::Commands::SetInstalled void perform_and_exit_ex(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const PathsPortFileProvider& provider, - BinaryCache& binary_cache, const CMakeVars::CMakeVarProvider& cmake_vars, ActionPlan action_plan, DryRun dry_run, @@ -112,7 +111,7 @@ namespace vcpkg::Commands::SetInstalled { compute_all_abis(paths, action_plan, cmake_vars, status_db); auto pkgsconfig_path = paths.original_cwd / *p_pkgsconfig; - auto pkgsconfig_contents = generate_nuget_packages_config(action_plan); + auto pkgsconfig_contents = generate_nuget_packages_config(action_plan, args.nuget_id_prefix.value_or("")); fs.write_contents(pkgsconfig_path, pkgsconfig_contents, VCPKG_LINE_INFO); msg::println(msgWroteNuGetPkgConfInfo, msg::path = pkgsconfig_path); } @@ -126,8 +125,8 @@ namespace vcpkg::Commands::SetInstalled track_install_plan(action_plan); - const auto summary = Install::perform( - args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder(), cmake_vars); + const auto summary = + Install::perform(args, action_plan, keep_going, paths, status_db, null_build_logs_recorder(), cmake_vars); if (keep_going == KeepGoing::YES && summary.failed()) { @@ -172,8 +171,6 @@ namespace vcpkg::Commands::SetInstalled print_default_triplet_warning(args); } - BinaryCache binary_cache{args, paths}; - const bool dry_run = Util::Sets::contains(options.switches, OPTION_DRY_RUN); const bool only_downloads = Util::Sets::contains(options.switches, OPTION_ONLY_DOWNLOADS); const KeepGoing keep_going = Util::Sets::contains(options.switches, OPTION_KEEP_GOING) || only_downloads @@ -203,8 +200,8 @@ namespace vcpkg::Commands::SetInstalled // We have a set of user-requested specs. // We need to know all the specs which are required to fulfill dependencies for those specs. // Therefore, we see what we would install into an empty installed tree, so we can use the existing code. - auto action_plan = - create_feature_install_plan(provider, *cmake_vars, specs, {}, {host_triplet, unsupported_port_action}); + auto action_plan = create_feature_install_plan( + provider, *cmake_vars, specs, {}, {host_triplet, paths.packages(), unsupported_port_action}); for (auto&& action : action_plan.install_actions) { @@ -216,7 +213,6 @@ namespace vcpkg::Commands::SetInstalled perform_and_exit_ex(args, paths, provider, - binary_cache, *cmake_vars, std::move(action_plan), dry_run ? DryRun::Yes : DryRun::No, diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index 055f200079..6b49dd4734 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -59,7 +59,6 @@ namespace vcpkg::Commands::Upgrade ? UnsupportedPortAction::Warn : UnsupportedPortAction::Error; - BinaryCache binary_cache{args, paths}; StatusParagraphs status_db = database_load_check(paths.get_filesystem(), paths.installed()); // Load ports from ports dirs @@ -87,7 +86,7 @@ namespace vcpkg::Commands::Upgrade var_provider, Util::fmap(outdated_packages, [](const Update::OutdatedPackage& package) { return package.spec; }), status_db, - {host_triplet, unsupported_port_action}); + {host_triplet, paths.packages(), unsupported_port_action}); } else { @@ -181,8 +180,11 @@ namespace vcpkg::Commands::Upgrade if (to_upgrade.empty()) Checks::exit_success(VCPKG_LINE_INFO); - action_plan = create_upgrade_plan( - provider, var_provider, to_upgrade, status_db, {host_triplet, unsupported_port_action}); + action_plan = create_upgrade_plan(provider, + var_provider, + to_upgrade, + status_db, + {host_triplet, paths.packages(), unsupported_port_action}); } Checks::check_exit(VCPKG_LINE_INFO, !action_plan.empty()); @@ -203,8 +205,8 @@ namespace vcpkg::Commands::Upgrade var_provider.load_tag_vars(action_plan, provider, host_triplet); - const InstallSummary summary = Install::perform( - args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder(), var_provider); + const InstallSummary summary = + Install::perform(args, action_plan, keep_going, paths, status_db, null_build_logs_recorder(), var_provider); if (keep_going == KeepGoing::YES) { diff --git a/src/vcpkg/dependencies.cpp b/src/vcpkg/dependencies.cpp index fa249698f0..cc57cc7ddd 100644 --- a/src/vcpkg/dependencies.cpp +++ b/src/vcpkg/dependencies.cpp @@ -463,8 +463,8 @@ namespace vcpkg , request_type(request_type) , build_options{} , feature_dependencies(std::move(dependencies)) - , build_failure_messages(std::move(build_failure_messages)) , host_triplet(host_triplet) + , build_failure_messages(std::move(build_failure_messages)) { } @@ -719,6 +719,12 @@ namespace vcpkg auto res = pgraph.serialize(options.randomizer); + for (auto&& action : res.install_actions) + { + if (action.source_control_file_and_location.has_value()) + action.package_dir = options.packages_dir / action.spec.dir(); + } + return res; } @@ -2047,16 +2053,23 @@ namespace vcpkg const std::vector& deps, const std::vector& overrides, const PackageSpec& toplevel, - Triplet host_triplet, - UnsupportedPortAction unsupported_port_action) + const CreateInstallPlanOptions& options) { - VersionedPackageGraph vpg(provider, bprovider, oprovider, var_provider, host_triplet); + VersionedPackageGraph vpg(provider, bprovider, oprovider, var_provider, options.host_triplet); for (auto&& o : overrides) { vpg.add_override(o.name, {o.version, o.port_version}); } vpg.add_roots(deps, toplevel); - return vpg.finalize_extract_plan(toplevel, unsupported_port_action); + auto ret = vpg.finalize_extract_plan(toplevel, options.unsupported_port_action); + if (auto plan = ret.get()) + { + for (auto&& action : plan->install_actions) + { + action.package_dir = options.packages_dir / action.spec.dir(); + } + } + return ret; } } diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index 53b2e02343..b77938f118 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -615,8 +615,9 @@ namespace vcpkg::Export::Prefab Debug::print( fmt::format("Exporting AAR and POM\n\tAAR path {}\n\tPOM path {}", exported_archive_path, pom_path)); - auto compress_result = compress_directory_to_zip( - paths.get_filesystem(), paths.get_tool_cache(), stdout_sink, package_directory, exported_archive_path); + ZipTool zip(paths.get_filesystem(), paths.get_tool_cache(), stdout_sink); + + auto compress_result = zip.compress_directory_to_zip(package_directory, exported_archive_path); if (!compress_result) { Checks::msg_exit_with_message( diff --git a/src/vcpkg/vcpkgcmdarguments.cpp b/src/vcpkg/vcpkgcmdarguments.cpp index a8938aacaa..9abe83f2a7 100644 --- a/src/vcpkg/vcpkgcmdarguments.cpp +++ b/src/vcpkg/vcpkgcmdarguments.cpp @@ -294,14 +294,14 @@ namespace vcpkg OVERLAY_PORTS_ARG, StabilityTag::Standard, args.cli_overlay_ports, - msg::format(msgSpecifyDirectoriesWhenSearching, msg::env_var = "VCPKG_OVERLAY_PORTS")); + msg::format(msgSpecifyDirectoriesWhenSearching, msg::env_var = OVERLAY_PORTS_ENV)); args.parser.parse_multi_option( OVERLAY_TRIPLETS_ARG, StabilityTag::Standard, args.cli_overlay_triplets, - msg::format(msgSpecifyDirectoriesContaining, msg::env_var = "VCPKG_OVERLAY_TRIPLETS")); + msg::format(msgSpecifyDirectoriesContaining, msg::env_var = OVERLAY_TRIPLETS_ENV)); args.parser.parse_multi_option( - BINARY_SOURCES_ARG, StabilityTag::Standard, args.binary_sources, msg::format(msgBinarySourcesArg)); + BINARY_SOURCES_ARG, StabilityTag::Standard, args.cli_binary_sources, msg::format(msgBinarySourcesArg)); args.parser.parse_multi_option(CMAKE_SCRIPT_ARG, StabilityTag::Standard, args.cmake_args); std::vector feature_flags; @@ -476,11 +476,23 @@ namespace vcpkg from_env(get_env, TRIPLET_ENV, triplet); from_env(get_env, HOST_TRIPLET_ENV, host_triplet); - vcpkg_root_dir_env = get_environment_variable(VCPKG_ROOT_DIR_ENV); + vcpkg_root_dir_env = get_env(VCPKG_ROOT_DIR_ENV); from_env(get_env, DOWNLOADS_ROOT_DIR_ENV, downloads_root_dir); - from_env(get_env, DEFAULT_VISUAL_STUDIO_PATH_ENV, default_visual_studio_path); from_env(get_env, ASSET_SOURCES_ENV, asset_sources_template_env); from_env(get_env, REGISTRIES_CACHE_DIR_ENV, registries_cache_dir); + from_env(get_env, DEFAULT_VISUAL_STUDIO_PATH_ENV, default_visual_studio_path); + from_env(get_env, BINARY_SOURCES_ENV, env_binary_sources); + from_env(get_env, ACTIONS_CACHE_URL_ENV, actions_cache_url); + from_env(get_env, ACTIONS_RUNTIME_TOKEN_ENV, actions_runtime_token); + from_env(get_env, NUGET_ID_PREFIX_ENV, nuget_id_prefix); + use_nuget_cache = get_env(VCPKG_USE_NUGET_CACHE_ENV).map([](const std::string& s) { + return Strings::case_insensitive_ascii_equals(s, "true") || s == "1"; + }); + from_env(get_env, VCPKG_NUGET_REPOSITORY_ENV, vcpkg_nuget_repository); + from_env(get_env, GITHUB_REPOSITORY_ENV, github_repository); + from_env(get_env, GITHUB_SERVER_URL_ENV, github_server_url); + from_env(get_env, GITHUB_REF_ENV, github_ref); + from_env(get_env, GITHUB_SHA_ENV, github_sha); // detect whether we are running in a CI environment for (auto&& ci_env_var : KNOWN_CI_VARIABLES) @@ -619,7 +631,7 @@ namespace vcpkg StringView option; bool is_inconsistent; } possible_inconsistencies[] = { - {BINARY_CACHING_FEATURE, BINARY_SOURCES_ARG, !binary_sources.empty() && !binary_caching.value_or(true)}, + {BINARY_CACHING_FEATURE, BINARY_SOURCES_ARG, !cli_binary_sources.empty() && !binary_caching.value_or(true)}, }; for (const auto& el : possible_inconsistencies) { @@ -726,6 +738,11 @@ namespace vcpkg constexpr StringLiteral VcpkgCmdArguments::OVERLAY_TRIPLETS_ARG; constexpr StringLiteral VcpkgCmdArguments::BINARY_SOURCES_ARG; + constexpr StringLiteral VcpkgCmdArguments::BINARY_SOURCES_ENV; + constexpr StringLiteral VcpkgCmdArguments::ACTIONS_CACHE_URL_ENV; + constexpr StringLiteral VcpkgCmdArguments::ACTIONS_RUNTIME_TOKEN_ENV; + constexpr StringLiteral VcpkgCmdArguments::NUGET_ID_PREFIX_ENV; + constexpr StringLiteral VcpkgCmdArguments::VCPKG_USE_NUGET_CACHE_ENV; constexpr StringLiteral VcpkgCmdArguments::DEBUG_SWITCH; constexpr StringLiteral VcpkgCmdArguments::DEBUG_ENV_SWITCH; From 3d3a43932e66f804ed6699d2b862bb054a0bdc41 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Sat, 6 May 2023 17:12:21 +0000 Subject: [PATCH 2/9] Fix macOS failure --- include/vcpkg/binarycaching.private.h | 1 - src/vcpkg/binarycaching.cpp | 154 +++++++++++++------------- 2 files changed, 78 insertions(+), 77 deletions(-) diff --git a/include/vcpkg/binarycaching.private.h b/include/vcpkg/binarycaching.private.h index a11285df2e..5123a286de 100644 --- a/include/vcpkg/binarycaching.private.h +++ b/include/vcpkg/binarycaching.private.h @@ -33,7 +33,6 @@ namespace vcpkg NugetReference make_nugetref(const InstallPlanAction& action, StringView prefix); - NuGetRepoInfo get_nuget_repo_info_from_env(); std::string generate_nuspec(const Path& package_dir, const InstallPlanAction& action, StringView id_prefix, diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index eaedbea4e3..e341751d7b 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -257,6 +257,21 @@ namespace std::vector m_dirs; }; + enum class ToRemove + { + nothing, + remove_always, + remove_on_fail, + }; + + struct ZipResource + { + ZipResource(Path&& p, ToRemove t) : path(std::move(p)), to_remove(t) { } + + Path path; + ToRemove to_remove; + }; + // This middleware class contains logic for BinaryProviders that operate on zip files. // Derived classes must implement: // - acquire_zips() @@ -269,7 +284,7 @@ namespace void prefetch(View actions, Span out_status) const override { const ElapsedTimer timer; - std::vector> zip_paths(actions.size(), nullopt); + std::vector> zip_paths(actions.size(), nullopt); acquire_zips(actions, zip_paths); std::vector jobs; @@ -279,7 +294,7 @@ namespace if (!zip_paths[i]) continue; const auto& pkg_path = actions[i]->package_dir.value_or_exit(VCPKG_LINE_INFO); clean_prepare_dir(m_fs, pkg_path); - jobs.push_back(m_zip.decompress_zip_archive_cmd(pkg_path, *zip_paths[i].get())); + jobs.push_back(m_zip.decompress_zip_archive_cmd(pkg_path, zip_paths[i].get()->path)); action_idxs.push_back(i); } @@ -291,14 +306,15 @@ namespace const auto& zip_path = zip_paths[i].value_or_exit(VCPKG_LINE_INFO); if (job_results[j]) { - Debug::print("Restored ", zip_path, '\n'); + Debug::print("Restored ", zip_path.path, '\n'); out_status[i] = RestoreResult::restored; } else { - Debug::print("Failed to decompress archive package: ", zip_path, '\n'); + Debug::print("Failed to decompress archive package: ", zip_path.path, '\n'); } - post_decompress(*actions[i], zip_path, job_results[j].has_value()); + + post_decompress(zip_path, job_results[j].has_value()); } size_t num_restored = std::count(out_status.begin(), out_status.end(), RestoreResult::restored); msg::println(msgRestoredPackagesFromVendor, @@ -307,19 +323,27 @@ namespace msg::value = vendor()); } + void post_decompress(const ZipResource& r, bool succeeded) const + { + if ((!succeeded && r.to_remove == ToRemove::remove_on_fail) || r.to_remove == ToRemove::remove_always) + { + m_fs.remove(r.path, IgnoreErrors{}); + } + } + RestoreResult try_restore(const InstallPlanAction& action) const override { // Note: this method is almost never called -- it will only be called if another provider promised to // restore a package but then failed at runtime auto p_action = &action; - Optional zip_path; - acquire_zips({&p_action, 1}, {&zip_path, 1}); - if (auto path = zip_path.get()) + Optional z; + acquire_zips({&p_action, 1}, {&z, 1}); + if (auto p = z.get()) { const auto& package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO); - auto cmd = m_zip.decompress_zip_archive_cmd(package_dir, *path); + auto cmd = m_zip.decompress_zip_archive_cmd(package_dir, p->path); auto result = decompress_in_parallel({&cmd, 1}); - post_decompress(action, *path, result[0].has_value()); + post_decompress(*p, result[0].has_value()); if (result[0]) { msg::println(msgRestoredPackage, msg::path = vendor()); @@ -330,12 +354,8 @@ namespace } // Download/find the zip files corresponding with a set of actions - virtual void acquire_zips(View actions, Span> out_zip_paths) const = 0; - - // Called for each zip file after `acquire_zips()`, should implement cleanup policies - virtual void post_decompress(const InstallPlanAction& action, - const Path& archive_path, - bool succeeded) const = 0; + virtual void acquire_zips(View actions, + Span> out_zips) const = 0; protected: ZipTool m_zip; @@ -349,7 +369,8 @@ namespace { } - void acquire_zips(View actions, Span> out_zip_paths) const override + void acquire_zips(View actions, + Span> out_zip_paths) const override { for (size_t i = 0; i < actions.size(); ++i) { @@ -357,20 +378,14 @@ namespace auto archive_path = m_dir / files_archive_subpath(abi_tag); if (m_fs.exists(archive_path, IgnoreErrors{})) { - out_zip_paths[i] = std::move(archive_path); + auto to_remove = actions[i]->build_options.purge_decompress_failure == PurgeDecompressFailure::YES + ? ToRemove::remove_on_fail + : ToRemove::nothing; + out_zip_paths[i].emplace(std::move(archive_path), to_remove); } } } - void post_decompress(const InstallPlanAction& action, const Path& archive_path, bool succeeded) const override - { - if (!succeeded && action.build_options.purge_decompress_failure == PurgeDecompressFailure::YES) - { - Debug::print("Failed to decompress archive package; purging: ", archive_path, '\n'); - m_fs.remove(archive_path, IgnoreErrors{}); - } - } - void precheck(View actions, Span cache_status) const override { for (size_t idx = 0; idx < actions.size(); ++idx) @@ -443,7 +458,8 @@ namespace { } - void acquire_zips(View actions, Span> out_zip_paths) const override + void acquire_zips(View actions, + Span> out_zip_paths) const override { std::vector> url_paths; for (size_t idx = 0; idx < actions.size(); ++idx) @@ -459,16 +475,11 @@ namespace { if (codes[i] == 200) { - out_zip_paths[i] = url_paths[i].second; + out_zip_paths[i].emplace(std::move(url_paths[i].second), ToRemove::remove_always); } } } - void post_decompress(const InstallPlanAction&, const Path& archive_path, bool) const override - { - m_fs.remove(archive_path, VCPKG_LINE_INFO); - } - void precheck(View actions, Span out_status) const override { std::vector urls; @@ -850,7 +861,8 @@ namespace return json.get()->get("archiveLocation")->string(VCPKG_LINE_INFO).to_string(); } - void acquire_zips(View actions, Span> out_zip_paths) const override + void acquire_zips(View actions, + Span> out_zip_paths) const override { std::vector> url_paths; std::vector url_indices; @@ -870,16 +882,11 @@ namespace { if (codes[i] == 200) { - out_zip_paths[url_indices[i]] = url_paths[i].second; + out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), ToRemove::remove_always); } } } - void post_decompress(const InstallPlanAction&, const Path& archive_path, bool) const override - { - m_fs.remove(archive_path, VCPKG_LINE_INFO); - } - void precheck(View, Span) const override { } StringView vendor() const override { return "GitHub Actions Cache"; } @@ -1006,7 +1013,8 @@ namespace return Strings::concat(prefix, abi, ".zip"); } - void acquire_zips(View actions, Span> out_zip_paths) const override + void acquire_zips(View actions, + Span> out_zip_paths) const override { for (size_t idx = 0; idx < actions.size(); ++idx) { @@ -1016,7 +1024,7 @@ namespace auto res = m_tool->download_file(make_object_path(m_prefix, abi), tmp); if (res) { - out_zip_paths[idx] = std::move(tmp); + out_zip_paths[idx].emplace(std::move(tmp), ToRemove::remove_always); } else { @@ -1025,11 +1033,6 @@ namespace } } - virtual void post_decompress(const InstallPlanAction&, const Path& archive_path, bool) const - { - m_fs.remove(archive_path, IgnoreErrors{}); - } - void precheck(View actions, Span cache_status) const override { for (size_t idx = 0; idx < actions.size(); ++idx) @@ -1804,6 +1807,32 @@ namespace vcpkg .value_or_exit(VCPKG_LINE_INFO); } + static NuGetRepoInfo get_nuget_repo_info_from_env(const VcpkgCmdArguments& args) + { + if (auto p = args.vcpkg_nuget_repository.get()) + { + get_global_metrics_collector().track_define(DefineMetric::VcpkgNugetRepository); + return {std::move(*p)}; + } + + auto gh_repo = get_environment_variable("GITHUB_REPOSITORY").value_or(""); + if (gh_repo.empty()) + { + return {}; + } + + auto gh_server = get_environment_variable("GITHUB_SERVER_URL").value_or(""); + if (gh_server.empty()) + { + return {}; + } + + get_global_metrics_collector().track_define(DefineMetric::GitHubRepository); + return {Strings::concat(gh_server, '/', gh_repo, ".git"), + get_environment_variable("GITHUB_REF").value_or(""), + get_environment_variable("GITHUB_SHA").value_or("")}; + } + static ExpectedL make_binary_providers(const VcpkgCmdArguments& args, const VcpkgPaths& paths) { BinaryProviders ret; @@ -1866,7 +1895,7 @@ namespace vcpkg s.nuget_prefix = args.nuget_id_prefix.value_or(""); if (!s.nuget_prefix.empty()) s.nuget_prefix.push_back('_'); s.use_nuget_cache = args.use_nuget_cache.value_or(false); - s.nuget_repo_info = get_nuget_repo_info_from_env(); + s.nuget_repo_info = get_nuget_repo_info_from_env(args); auto& fs = paths.get_filesystem(); auto& tools = paths.get_tool_cache(); @@ -2396,33 +2425,6 @@ std::string vcpkg::format_version_for_nugetref(StringView version, StringView ab return Strings::concat("0.0.0-vcpkg", abi_tag); } -NuGetRepoInfo vcpkg::get_nuget_repo_info_from_env() -{ - auto vcpkg_nuget_repository = get_environment_variable("VCPKG_NUGET_REPOSITORY"); - if (auto p = vcpkg_nuget_repository.get()) - { - get_global_metrics_collector().track_define(DefineMetric::VcpkgNugetRepository); - return {std::move(*p)}; - } - - auto gh_repo = get_environment_variable("GITHUB_REPOSITORY").value_or(""); - if (gh_repo.empty()) - { - return {}; - } - - auto gh_server = get_environment_variable("GITHUB_SERVER_URL").value_or(""); - if (gh_server.empty()) - { - return {}; - } - - get_global_metrics_collector().track_define(DefineMetric::GitHubRepository); - return {Strings::concat(gh_server, '/', gh_repo, ".git"), - get_environment_variable("GITHUB_REF").value_or(""), - get_environment_variable("GITHUB_SHA").value_or("")}; -} - std::string vcpkg::generate_nuspec(const Path& package_dir, const InstallPlanAction& action, StringView id_prefix, From 6a4a2bb8a874073a45ef0a21ec982712397e2fa0 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Sat, 6 May 2023 22:18:05 +0000 Subject: [PATCH 3/9] Remove try_restore as defunct --- include/vcpkg/binarycaching.h | 14 ++-- include/vcpkg/fwd/binarycaching.h | 1 - src/vcpkg-test/binarycaching.cpp | 11 +--- src/vcpkg/binarycaching.cpp | 102 ++++++------------------------ src/vcpkg/commands.install.cpp | 5 +- 5 files changed, 27 insertions(+), 106 deletions(-) diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index 6583e672d0..6253afb7bf 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -82,18 +82,13 @@ namespace vcpkg { virtual ~IReadBinaryProvider() = default; - /// Attempts to restore the package referenced by `action` into the packages directory. - /// - /// Prerequisite: action has a package_abi() - virtual RestoreResult try_restore(const InstallPlanAction& action) const = 0; - /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for executing /// `actions`. /// /// IBinaryProvider should set out_status[i] to RestoreResult::restored for each fetched package. /// /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size() - virtual void prefetch(View actions, Span out_status) const = 0; + virtual void fetch(View actions, Span out_status) const = 0; /// Checks whether the `actions` are present in the cache, without restoring them. /// @@ -182,12 +177,11 @@ namespace vcpkg ReadOnlyBinaryCache(BinaryProviders&& providers); ReadOnlyBinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info); - /// Attempts to restore the package referenced by `action` into the packages directory. - RestoreResult try_restore(const InstallPlanAction& action); - /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for /// executing `actions`. - void prefetch(View actions); + void fetch(View actions); + + bool is_restored(const InstallPlanAction& ipa) const; /// Checks whether the `actions` are present in the cache, without restoring them. Used by CI to determine /// missing packages. diff --git a/include/vcpkg/fwd/binarycaching.h b/include/vcpkg/fwd/binarycaching.h index 51efde3b91..205207f647 100644 --- a/include/vcpkg/fwd/binarycaching.h +++ b/include/vcpkg/fwd/binarycaching.h @@ -22,7 +22,6 @@ namespace vcpkg restored, // the cache contains the ABI and it has been restored to the packages tree }; - struct CacheStatus; struct IReadBinaryProvider; struct IWriteBinaryProvider; struct BinaryCache; diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 875717f9b2..01648827fe 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -19,13 +19,7 @@ using namespace vcpkg; struct KnowNothingBinaryProvider : IReadBinaryProvider { - RestoreResult try_restore(const InstallPlanAction& action) const override - { - CHECK(action.has_package_abi()); - return RestoreResult::unavailable; - } - - void prefetch(View actions, Span out_status) const override + void fetch(View actions, Span out_status) const override { REQUIRE(actions.size() == out_status.size()); for (size_t idx = 0; idx < out_status.size(); ++idx) @@ -376,8 +370,7 @@ Version: 1.5 ipa_without_abi.package_dir = "pkgs/someheadpackage"; // test that the binary cache does the right thing. See also CHECKs etc. in KnowNothingBinaryProvider - CHECK(uut.try_restore(ipa_without_abi) == RestoreResult::unavailable); - uut.prefetch(install_plan); // should have no effects + uut.fetch(install_plan); // should have no effects } TEST_CASE ("XmlSerializer", "[XmlSerializer]") diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index e341751d7b..7c52fa20d7 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -281,7 +281,7 @@ namespace { ZipReadBinaryProvider(ZipTool zip, Filesystem& fs) : m_zip(std::move(zip)), m_fs(fs) { } - void prefetch(View actions, Span out_status) const override + void fetch(View actions, Span out_status) const override { const ElapsedTimer timer; std::vector> zip_paths(actions.size(), nullopt); @@ -331,28 +331,6 @@ namespace } } - RestoreResult try_restore(const InstallPlanAction& action) const override - { - // Note: this method is almost never called -- it will only be called if another provider promised to - // restore a package but then failed at runtime - auto p_action = &action; - Optional z; - acquire_zips({&p_action, 1}, {&z, 1}); - if (auto p = z.get()) - { - const auto& package_dir = action.package_dir.value_or_exit(VCPKG_LINE_INFO); - auto cmd = m_zip.decompress_zip_archive_cmd(package_dir, p->path); - auto result = decompress_in_parallel({&cmd, 1}); - post_decompress(*p, result[0].has_value()); - if (result[0]) - { - msg::println(msgRestoredPackage, msg::path = vendor()); - return RestoreResult::restored; - } - } - return RestoreResult::unavailable; - } - // Download/find the zip files corresponding with a set of actions virtual void acquire_zips(View actions, Span> out_zips) const = 0; @@ -701,15 +679,12 @@ namespace return std::move(xml.buf); } - // Individual package restore is too expensive with NuGet, so it is not implemented - RestoreResult try_restore(const InstallPlanAction&) const override { return RestoreResult::unavailable; } - // Prechecking is too expensive with NuGet, so it is not implemented void precheck(View, Span) const override { } StringView vendor() const override { return "NuGet"; } - void prefetch(View actions, Span out_status) const override + void fetch(View actions, Span out_status) const override { auto packages_config = m_buildtrees / "packages.config"; auto refs = @@ -2005,55 +1980,7 @@ namespace vcpkg ReadOnlyBinaryCache::ReadOnlyBinaryCache(BinaryProviders&& providers) : m_config(std::move(providers)) { } - RestoreResult ReadOnlyBinaryCache::try_restore(const InstallPlanAction& action) - { - const auto abi = action.package_abi().get(); - if (!abi) - { - // e.g. this is a `--head` package - return RestoreResult::unavailable; - } - - auto& cache_status = m_status[*abi]; - if (cache_status.is_restored()) - { - return RestoreResult::restored; - } - - const auto available = cache_status.get_available_provider(); - if (available) - { - switch (available->try_restore(action)) - { - case RestoreResult::unavailable: - // Even though that provider thought it had it, it didn't; perhaps - // due to intermittent network problems etc. - // Try other providers below - break; - case RestoreResult::restored: cache_status.mark_restored(); return RestoreResult::restored; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - for (auto&& provider : m_config.read) - { - if (provider.get() == available || cache_status.is_unavailable(provider.get())) - { - continue; // this one was already tried or knows the package is unavailable - } - - switch (provider->try_restore(action)) - { - case RestoreResult::restored: cache_status.mark_restored(); return RestoreResult::restored; - case RestoreResult::unavailable: cache_status.mark_unavailable(provider.get()); break; - default: Checks::unreachable(VCPKG_LINE_INFO); - } - } - - return RestoreResult::unavailable; - } - - void ReadOnlyBinaryCache::prefetch(View actions) + void ReadOnlyBinaryCache::fetch(View actions) { std::vector action_ptrs; std::vector restores; @@ -2067,19 +1994,19 @@ namespace vcpkg { if (actions[i].package_abi()) { - CacheStatus* status = &m_status[*actions[i].package_abi().get()]; - if (status->should_attempt_restore(provider.get())) + CacheStatus& status = m_status[*actions[i].package_abi().get()]; + if (status.should_attempt_restore(provider.get())) { action_ptrs.push_back(&actions[i]); restores.push_back(RestoreResult::unavailable); - statuses.push_back(status); + statuses.push_back(&status); } } } if (action_ptrs.empty()) continue; ElapsedTimer timer; - provider->prefetch(action_ptrs, restores); + provider->fetch(action_ptrs, restores); size_t num_restored = 0; for (size_t i = 0; i < restores.size(); ++i) { @@ -2100,6 +2027,16 @@ namespace vcpkg } } + bool ReadOnlyBinaryCache::is_restored(const InstallPlanAction& action) const + { + if (auto abi = action.package_abi().get()) + { + auto it = m_status.find(*abi); + if (it != m_status.end()) return it->second.is_restored(); + } + return false; + } + std::vector ReadOnlyBinaryCache::precheck(View actions) { std::vector statuses = Util::fmap(actions, [this](const auto& action) { @@ -2141,10 +2078,9 @@ namespace vcpkg } } - std::vector ret = Util::fmap(statuses, [](CacheStatus* s) { + return Util::fmap(statuses, [](CacheStatus* s) { return s->get_available_provider() ? CacheAvailability::available : CacheAvailability::unavailable; }); - return ret; } BinaryCache::BinaryCache(RemoveFilesystem& fs) : m_fs(fs) { } @@ -2171,7 +2107,7 @@ namespace vcpkg bool restored = m_status[*action.package_abi().get()].is_restored(); // Purge all status information on push_success (cache invalidation) // - push_success may delete packages/ (invalidate restore) - // - push_success may make the package available from providers (invalidate valid) + // - push_success may make the package available from providers (invalidate unavailable) m_status.erase(*action.package_abi().get()); if (!restored && !m_config.write.empty()) { diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 07215ed7f7..a68522dd0d 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -326,8 +326,7 @@ namespace vcpkg if (plan_type == InstallPlanType::BUILD_AND_INSTALL) { std::unique_ptr bcf; - auto restore = binary_cache.try_restore(action); - if (restore == RestoreResult::restored) + if (binary_cache.is_restored(action)) { auto maybe_bcf = Paragraphs::try_load_cached_package( fs, action.package_dir.value_or_exit(VCPKG_LINE_INFO), action.spec); @@ -546,7 +545,7 @@ namespace vcpkg } compute_all_abis(paths, action_plan, var_provider, status_db); - binary_cache.prefetch(action_plan.install_actions); + binary_cache.fetch(action_plan.install_actions); for (auto&& action : action_plan.install_actions) { TrackedPackageInstallGuard this_install(action_index++, action_count, results, action); From cda6d76345f2293ea2b5c258dee3d64abbc34054 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Sat, 6 May 2023 22:30:19 +0000 Subject: [PATCH 4/9] Apply REQUIRE_LINES --- include/vcpkg-test/util.h | 8 +++ include/vcpkg/binarycaching.private.h | 2 +- src/vcpkg-test/binarycaching.cpp | 56 ++++++----------- src/vcpkg-test/dependencies.cpp | 87 --------------------------- src/vcpkg-test/util.cpp | 83 +++++++++++++++++++++++++ src/vcpkg/binarycaching.cpp | 2 +- 6 files changed, 112 insertions(+), 126 deletions(-) diff --git a/include/vcpkg-test/util.h b/include/vcpkg-test/util.h index 7a34860682..6cb3e964ed 100644 --- a/include/vcpkg-test/util.h +++ b/include/vcpkg-test/util.h @@ -177,4 +177,12 @@ namespace vcpkg::Test void check_json_eq_ordered(const Json::Array& l, const Json::Array& r); const Path& base_temporary_directory() noexcept; + + Optional diff_lines(StringView a, StringView b); } + +#define REQUIRE_LINES(a, b) \ + do \ + { \ + if (auto delta = ::vcpkg::Test::diff_lines((a), (b))) FAIL(*delta.get()); \ + } while (0) diff --git a/include/vcpkg/binarycaching.private.h b/include/vcpkg/binarycaching.private.h index 5123a286de..0987e06381 100644 --- a/include/vcpkg/binarycaching.private.h +++ b/include/vcpkg/binarycaching.private.h @@ -36,5 +36,5 @@ namespace vcpkg std::string generate_nuspec(const Path& package_dir, const InstallPlanAction& action, StringView id_prefix, - NuGetRepoInfo repo_info); + const NuGetRepoInfo& repo_info); } diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 01648827fe..5cadcc3e5d 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -168,18 +168,6 @@ TEST_CASE ("CacheStatus operations", "[BinaryCache]") REQUIRE(assignee.is_restored()); } -#define REQUIRE_EQUAL_TEXT(lhs, rhs) \ - { \ - auto lhs_lines = Strings::split((lhs), '\n'); \ - auto rhs_lines = Strings::split((rhs), '\n'); \ - for (size_t i = 0; i < lhs_lines.size() && i < rhs_lines.size(); ++i) \ - { \ - INFO("on line: " << i); \ - REQUIRE(lhs_lines[i] == rhs_lines[i]); \ - } \ - REQUIRE(lhs_lines.size() == rhs_lines.size()); \ - } - TEST_CASE ("format_version_for_nugetref semver-ish", "[format_version_for_nugetref]") { REQUIRE(format_version_for_nugetref("0.0.0", "abitag") == "0.0.0-vcpkgabitag"); @@ -255,9 +243,8 @@ Build-Depends: bzip REQUIRE(ref.nupkg_filename() == "zlib2_x64-windows.1.5.0-vcpkgpackageabi.nupkg"); - { - auto nuspec = generate_nuspec(pkgPath, ipa, "", {}); - std::string expected = R"( + REQUIRE_LINES(generate_nuspec(pkgPath, ipa, "", {}), + R"( zlib2_x64-windows 1.5.0-vcpkgpackageabi @@ -276,16 +263,14 @@ Features: a, b - + -)"; - REQUIRE_EQUAL_TEXT(nuspec, expected); - } +)"); - { - auto nuspec = generate_nuspec(pkgPath, ipa, "", {"urlvalue"}); - std::string expected = R"( + REQUIRE_LINES(generate_nuspec(pkgPath, ipa, "", {"urlvalue"}), + R"( zlib2_x64-windows 1.5.0-vcpkgpackageabi @@ -305,15 +290,13 @@ Features: a, b - + -)"; - REQUIRE_EQUAL_TEXT(nuspec, expected); - } - { - auto nuspec = generate_nuspec(pkgPath, ipa, "", {"urlvalue", "branchvalue", "commitvalue"}); - std::string expected = R"( +)"); + REQUIRE_LINES(generate_nuspec(pkgPath, ipa, "", {"urlvalue", "branchvalue", "commitvalue"}), + R"( zlib2_x64-windows 1.5.0-vcpkgpackageabi @@ -333,12 +316,11 @@ Features: a, b - + -)"; - REQUIRE_EQUAL_TEXT(nuspec, expected); - } +)"); } TEST_CASE ("Provider nullptr checks", "[BinaryCache]") @@ -471,7 +453,7 @@ Description: a spiffy compression library wrapper plan.install_actions[1].abi_info.get()->package_abi = "packageabi2"; packageconfig = generate_nuget_packages_config(plan, ""); - REQUIRE(packageconfig == R"( + REQUIRE_LINES(packageconfig, R"( diff --git a/src/vcpkg-test/dependencies.cpp b/src/vcpkg-test/dependencies.cpp index 04b1ed9ec1..3992910493 100644 --- a/src/vcpkg-test/dependencies.cpp +++ b/src/vcpkg-test/dependencies.cpp @@ -1087,93 +1087,6 @@ TEST_CASE ("version install diamond date", "[versionplan]") check_name_and_version(install_plan.install_actions[1], "b", {"2020-01-03", 0}); check_name_and_version(install_plan.install_actions[2], "a", {"2020-01-03", 0}); } -static Optional diff_lines(StringView a, StringView b) -{ - auto lines_a = Strings::split_keep_empty(a, '\n'); - auto lines_b = Strings::split_keep_empty(b, '\n'); - - std::vector> edits; - auto& first_row = edits.emplace_back(); - first_row.resize(lines_b.size() + 1); - std::iota(first_row.begin(), first_row.end(), 0); - for (size_t i = 0; i < lines_a.size(); ++i) - { - edits.emplace_back().resize(lines_b.size() + 1); - edits[i + 1][0] = edits[i][0] + 1; - for (size_t j = 0; j < lines_b.size(); ++j) - { - size_t p = edits[i + 1][j] + 1; - size_t m = edits[i][j + 1] + 1; - if (m < p) p = m; - if (lines_a[i] == lines_b[j] && edits[i][j] < p) p = edits[i][j]; - edits[i + 1][j + 1] = p; - } - } - - size_t i = lines_a.size(); - size_t j = lines_b.size(); - if (edits[i][j] == 0) return nullopt; - - std::vector lines; - - while (i > 0 && j > 0) - { - if (edits[i][j] == edits[i - 1][j - 1] && lines_a[i - 1] == lines_b[j - 1]) - { - --j; - --i; - lines.emplace_back(" " + lines_a[i]); - } - else if (edits[i][j] == edits[i - 1][j] + 1) - { - --i; - lines.emplace_back("-" + lines_a[i]); - } - else - { - --j; - lines.emplace_back("+" + lines_b[j]); - } - } - for (; i > 0; --i) - { - lines.emplace_back("-" + lines_a[i - 1]); - } - for (; j > 0; --j) - { - lines.emplace_back("+" + lines_b[j - 1]); - } - std::string ret; - for (auto it = lines.rbegin(); it != lines.rend(); ++it) - { - ret.append(*it); - ret.push_back('\n'); - } - return ret; -} - -#define REQUIRE_LINES(a, b) \ - do \ - { \ - if (auto delta = diff_lines((a), (b))) FAIL(*delta.get()); \ - } while (0) - -TEST_CASE ("diff algorithm", "[diff]") -{ - CHECK(!diff_lines("hello", "hello")); - CHECK(!diff_lines("hello\n", "hello\n")); - CHECK(!diff_lines("hello\n\nworld", "hello\n\nworld")); - { - auto a = diff_lines("hello\na\nworld", "hello\nworld"); - REQUIRE(a); - CHECK(*a.get() == " hello\n-a\n world\n"); - } - { - auto a = diff_lines("hello\nworld", "hello\na\nworld"); - REQUIRE(a); - CHECK(*a.get() == " hello\n+a\n world\n"); - } -} TEST_CASE ("version install scheme failure", "[versionplan]") { diff --git a/src/vcpkg-test/util.cpp b/src/vcpkg-test/util.cpp index 68790e03f2..1daa5feceb 100644 --- a/src/vcpkg-test/util.cpp +++ b/src/vcpkg-test/util.cpp @@ -266,4 +266,87 @@ namespace vcpkg::Test std::string path = "$"; check_json_eq(l, r, path, true); } + + Optional diff_lines(StringView a, StringView b) + { + auto lines_a = Strings::split_keep_empty(a, '\n'); + auto lines_b = Strings::split_keep_empty(b, '\n'); + + std::vector> edits; + auto& first_row = edits.emplace_back(); + first_row.resize(lines_b.size() + 1); + std::iota(first_row.begin(), first_row.end(), 0); + for (size_t i = 0; i < lines_a.size(); ++i) + { + edits.emplace_back().resize(lines_b.size() + 1); + edits[i + 1][0] = edits[i][0] + 1; + for (size_t j = 0; j < lines_b.size(); ++j) + { + size_t p = edits[i + 1][j] + 1; + size_t m = edits[i][j + 1] + 1; + if (m < p) p = m; + if (lines_a[i] == lines_b[j] && edits[i][j] < p) p = edits[i][j]; + edits[i + 1][j + 1] = p; + } + } + + size_t i = lines_a.size(); + size_t j = lines_b.size(); + if (edits[i][j] == 0) return nullopt; + + std::vector lines; + + while (i > 0 && j > 0) + { + if (edits[i][j] == edits[i - 1][j - 1] && lines_a[i - 1] == lines_b[j - 1]) + { + --j; + --i; + lines.emplace_back(" " + lines_a[i]); + } + else if (edits[i][j] == edits[i - 1][j] + 1) + { + --i; + lines.emplace_back("-" + lines_a[i]); + } + else + { + --j; + lines.emplace_back("+" + lines_b[j]); + } + } + for (; i > 0; --i) + { + lines.emplace_back("-" + lines_a[i - 1]); + } + for (; j > 0; --j) + { + lines.emplace_back("+" + lines_b[j - 1]); + } + std::string ret; + for (auto it = lines.rbegin(); it != lines.rend(); ++it) + { + ret.append(*it); + ret.push_back('\n'); + } + return ret; + } +} + +TEST_CASE ("diff algorithm", "[diff]") +{ + using namespace vcpkg::Test; + CHECK(!diff_lines("hello", "hello")); + CHECK(!diff_lines("hello\n", "hello\n")); + CHECK(!diff_lines("hello\n\nworld", "hello\n\nworld")); + { + auto a = diff_lines("hello\na\nworld", "hello\nworld"); + REQUIRE(a); + CHECK(*a.get() == " hello\n-a\n world\n"); + } + { + auto a = diff_lines("hello\nworld", "hello\na\nworld"); + REQUIRE(a); + CHECK(*a.get() == " hello\n+a\n world\n"); + } } diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 7c52fa20d7..7a76ef9f34 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -2364,7 +2364,7 @@ std::string vcpkg::format_version_for_nugetref(StringView version, StringView ab std::string vcpkg::generate_nuspec(const Path& package_dir, const InstallPlanAction& action, StringView id_prefix, - NuGetRepoInfo rinfo) + const NuGetRepoInfo& rinfo) { auto& spec = action.spec; auto& scf = *action.source_control_file_and_location.value_or_exit(VCPKG_LINE_INFO).source_control_file; From e2a07dfc89040b4f18e4234d002e206aa15701bb Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Sat, 6 May 2023 22:52:07 +0000 Subject: [PATCH 5/9] Rename Install::perform -> Install::execute_plan to better match behavior. --- include/vcpkg/commands.install.h | 14 +++++++------- src/vcpkg/commands.ci.cpp | 7 ++++--- src/vcpkg/commands.install.cpp | 27 +++++++++++++++------------ src/vcpkg/commands.set-installed.cpp | 10 ++++++++-- src/vcpkg/commands.upgrade.cpp | 7 +++++-- 5 files changed, 39 insertions(+), 26 deletions(-) diff --git a/include/vcpkg/commands.install.h b/include/vcpkg/commands.install.h index 164c43cef4..9de970f3a1 100644 --- a/include/vcpkg/commands.install.h +++ b/include/vcpkg/commands.install.h @@ -92,13 +92,13 @@ namespace vcpkg const Filesystem& fs, const InstalledPaths& installed); - InstallSummary perform(const VcpkgCmdArguments& args, - ActionPlan& action_plan, - const KeepGoing keep_going, - const VcpkgPaths& paths, - StatusParagraphs& status_db, - const IBuildLogsRecorder& build_logs_recorder, - const CMakeVars::CMakeVarProvider& var_provider); + InstallSummary execute_plan(const VcpkgCmdArguments& args, + ActionPlan& action_plan, + const KeepGoing keep_going, + const VcpkgPaths& paths, + StatusParagraphs& status_db, + BinaryCache& binary_cache, + const IBuildLogsRecorder& build_logs_recorder); void perform_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths, diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index d14a4dee4c..6611d1829a 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -30,7 +30,7 @@ #include #include -#include +#include using namespace vcpkg; @@ -514,8 +514,9 @@ namespace vcpkg::Commands::CI { msg::println_warning(msgCISkipInstallation, msg::list = Strings::join(", ", already_installed)); } - auto summary = Install::perform( - args, action_plan, KeepGoing::YES, paths, status_db, build_logs_recorder, var_provider); + binary_cache.fetch(action_plan.install_actions); + auto summary = Install::execute_plan( + args, action_plan, KeepGoing::YES, paths, status_db, binary_cache, build_logs_recorder); for (auto&& result : summary.results) { diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index a68522dd0d..861690970f 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -514,13 +514,13 @@ namespace vcpkg TrackedPackageInstallGuard& operator=(const TrackedPackageInstallGuard&) = delete; }; - InstallSummary Install::perform(const VcpkgCmdArguments& args, - ActionPlan& action_plan, - const KeepGoing keep_going, - const VcpkgPaths& paths, - StatusParagraphs& status_db, - const IBuildLogsRecorder& build_logs_recorder, - const CMakeVars::CMakeVarProvider& var_provider) + InstallSummary Install::execute_plan(const VcpkgCmdArguments& args, + ActionPlan& action_plan, + const KeepGoing keep_going, + const VcpkgPaths& paths, + StatusParagraphs& status_db, + BinaryCache& binary_cache, + const IBuildLogsRecorder& build_logs_recorder) { const ElapsedTimer timer; std::vector results; @@ -536,7 +536,6 @@ namespace vcpkg results.back().build_result.emplace(BuildResult::REMOVED); } - BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); for (auto&& action : action_plan.already_installed) { results.emplace_back(action); @@ -544,8 +543,6 @@ namespace vcpkg perform_install_plan_action(args, paths, action, status_db, binary_cache, build_logs_recorder)); } - compute_all_abis(paths, action_plan, var_provider, status_db); - binary_cache.fetch(action_plan.install_actions); for (auto&& action : action_plan.install_actions) { TrackedPackageInstallGuard this_install(action_index++, action_count, results, action); @@ -1261,6 +1258,10 @@ namespace vcpkg fs.write_contents(pkgsconfig_path, pkgsconfig_contents, VCPKG_LINE_INFO); msg::println(msgWroteNuGetPkgConfInfo, msg::path = pkgsconfig_path); } + else if (!dry_run) + { + compute_all_abis(paths, action_plan, var_provider, status_db); + } if (dry_run) { @@ -1271,8 +1272,10 @@ namespace vcpkg track_install_plan(action_plan); - const InstallSummary summary = - Install::perform(args, action_plan, keep_going, paths, status_db, null_build_logs_recorder(), var_provider); + BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); + binary_cache.fetch(action_plan.install_actions); + const InstallSummary summary = Install::execute_plan( + args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder()); if (keep_going == KeepGoing::YES) { diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 652bac6d51..4f644b75a5 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -123,6 +123,10 @@ namespace vcpkg::Commands::SetInstalled fs.write_contents(pkgsconfig_path, pkgsconfig_contents, VCPKG_LINE_INFO); msg::println(msgWroteNuGetPkgConfInfo, msg::path = pkgsconfig_path); } + else if (dry_run != DryRun::Yes) + { + compute_all_abis(paths, action_plan, cmake_vars, status_db); + } if (dry_run == DryRun::Yes) { @@ -133,8 +137,10 @@ namespace vcpkg::Commands::SetInstalled track_install_plan(action_plan); - const auto summary = - Install::perform(args, action_plan, keep_going, paths, status_db, null_build_logs_recorder(), cmake_vars); + BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); + binary_cache.fetch(action_plan.install_actions); + const auto summary = Install::execute_plan( + args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder()); if (keep_going == KeepGoing::YES && summary.failed()) { diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index 6b49dd4734..4522db01cc 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -205,8 +205,11 @@ namespace vcpkg::Commands::Upgrade var_provider.load_tag_vars(action_plan, provider, host_triplet); - const InstallSummary summary = - Install::perform(args, action_plan, keep_going, paths, status_db, null_build_logs_recorder(), var_provider); + BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); + binary_cache.fetch(action_plan.install_actions); + compute_all_abis(paths, action_plan, var_provider, status_db); + const InstallSummary summary = Install::execute_plan( + args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder()); if (keep_going == KeepGoing::YES) { From c323335f272dd087bb2d205a8fcbecd5d7b2b637 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 9 May 2023 21:32:07 +0000 Subject: [PATCH 6/9] Address some PR comments --- .vscode/settings.json | 70 +++++- include/vcpkg/archives.h | 7 +- include/vcpkg/base/files.h | 24 +- include/vcpkg/base/message-data.inc.h | 35 ++- include/vcpkg/binarycaching.h | 18 +- src/vcpkg-test/binarycaching.cpp | 5 +- src/vcpkg/archives.cpp | 10 +- src/vcpkg/base/files.cpp | 18 +- src/vcpkg/binarycaching.cpp | 318 +++++++++++++++----------- src/vcpkg/commands.build.cpp | 2 +- src/vcpkg/commands.ci.cpp | 2 +- src/vcpkg/commands.install.cpp | 4 +- src/vcpkg/commands.set-installed.cpp | 3 +- src/vcpkg/commands.upgrade.cpp | 2 +- src/vcpkg/export.prefab.cpp | 3 +- 15 files changed, 341 insertions(+), 180 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index b9a205d37e..79a72d6597 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,5 +31,73 @@ "source.organizeImports": true }, "eslint.format.enable": true, - "files.eol": "\n" + "files.eol": "\n", + "files.associations": { + "any": "cpp", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "cctype": "cpp", + "chrono": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "codecvt": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "regex": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "future": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "shared_mutex": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "cinttypes": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "variant": "cpp" + } } diff --git a/include/vcpkg/archives.h b/include/vcpkg/archives.h index fb02b8b7e3..c3b15706af 100644 --- a/include/vcpkg/archives.h +++ b/include/vcpkg/archives.h @@ -32,11 +32,12 @@ namespace vcpkg struct ZipTool { - // On Windows, acquires 7zip in the constructor - ZipTool(RemoveFilesystem& fs, const ToolCache& tools, MessageSink& status_sink); + static ExpectedL make(Filesystem& fs, const ToolCache& tools, MessageSink& status_sink); private: - RemoveFilesystem* fs; + ZipTool() = default; + + Filesystem* fs = nullptr; #if defined _WIN32 Path seven_zip; #endif diff --git a/include/vcpkg/base/files.h b/include/vcpkg/base/files.h index d6ae78245a..12eb454138 100644 --- a/include/vcpkg/base/files.h +++ b/include/vcpkg/base/files.h @@ -107,23 +107,10 @@ namespace vcpkg virtual ExpectedL> read_lines(const Path& file_path) const = 0; protected: - ~ILineReader() = default; + ~ILineReader(); }; - struct RemoveFilesystem - { - virtual bool remove(const Path& target, std::error_code& ec) = 0; - bool remove(const Path& target, LineInfo li); - - virtual void remove_all(const Path& base, std::error_code& ec, Path& failure_point) = 0; - void remove_all(const Path& base, std::error_code& ec); - void remove_all(const Path& base, LineInfo li); - - protected: - ~RemoveFilesystem() = default; - }; - - struct Filesystem : ILineReader, RemoveFilesystem + struct Filesystem : ILineReader { virtual std::string read_contents(const Path& file_path, std::error_code& ec) const = 0; std::string read_contents(const Path& file_path, LineInfo li) const; @@ -178,6 +165,13 @@ namespace vcpkg StringLiteral temp_suffix, std::error_code& ec) = 0; + virtual bool remove(const Path& target, std::error_code& ec) = 0; + bool remove(const Path& target, LineInfo li); + + virtual void remove_all(const Path& base, std::error_code& ec, Path& failure_point) = 0; + void remove_all(const Path& base, std::error_code& ec); + void remove_all(const Path& base, LineInfo li); + void remove_all_inside(const Path& base, std::error_code& ec, Path& failure_point); void remove_all_inside(const Path& base, std::error_code& ec); void remove_all_inside(const Path& base, LineInfo li); diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 50acf4555a..cf2f857d7a 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -260,6 +260,7 @@ DECLARE_MESSAGE(BaselineMissingDefault, "", "The baseline.json from commit `\"{commit_sha}\"` in the repo {url} was invalid (did not " "contain a \"default\" field).") +DECLARE_MESSAGE(BinaryCacheVendorHTTP, (), "", "HTTP servers") DECLARE_MESSAGE(BinarySourcesArg, (), "", "Add sources for binary caching. See 'vcpkg help binarycaching'.") DECLARE_MESSAGE(BinaryWithInvalidArchitecture, (msg::path, msg::expected, msg::actual), @@ -2230,12 +2231,34 @@ DECLARE_MESSAGE(ResponseFileCode, "Explains to the user that they can use response files on the command line, 'response_file' must " "have no spaces and be a legal file name.", "@response_file") -DECLARE_MESSAGE(RestoredPackage, (msg::path), "", "Restored package from \"{path}\"") -DECLARE_MESSAGE( - RestoredPackagesFromVendor, - (msg::count, msg::elapsed, msg::value), - "{value} may be either a 'vendor' like 'Azure' or 'NuGet', or a file path like C:\\example or /usr/example", - "Restored {count} package(s) from {value} in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromAWS, + (msg::count, msg::elapsed), + "", + "Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromCOS, + (msg::count, msg::elapsed), + "", + "Restored {count} package(s) from COS in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromFiles, + (msg::count, msg::elapsed, msg::path), + "", + "Restored {count} package(s) from {path} in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromGCS, + (msg::count, msg::elapsed), + "", + "Restored {count} package(s) from GCS in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromGHA, + (msg::count, msg::elapsed), + "", + "Restored {count} package(s) from GitHub Actions Cache in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromHTTP, + (msg::count, msg::elapsed), + "", + "Restored {count} package(s) from HTTP servers in {elapsed}. Use --debug to see more details.") +DECLARE_MESSAGE(RestoredPackagesFromNuGet, + (msg::count, msg::elapsed), + "", + "Restored {count} package(s) from NuGet in {elapsed}. Use --debug to see more details.") DECLARE_MESSAGE(ResultsHeader, (), "Displayed before a list of installation results.", "RESULTS") DECLARE_MESSAGE(ScriptAssetCacheRequiresScript, (), diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index 6253afb7bf..d56c7d7303 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -98,8 +99,8 @@ namespace vcpkg /// Prerequisites: actions[i].package_abi(), out_status.size() == actions.size() virtual void precheck(View actions, Span out_status) const = 0; - /// A user-visible vendor string identifying this provider - virtual StringView vendor() const = 0; + virtual LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const = 0; }; struct UrlTemplate @@ -175,7 +176,6 @@ namespace vcpkg { ReadOnlyBinaryCache() = default; ReadOnlyBinaryCache(BinaryProviders&& providers); - ReadOnlyBinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info); /// Gives the IBinaryProvider an opportunity to batch any downloading or server communication for /// executing `actions`. @@ -196,16 +196,20 @@ namespace vcpkg struct BinaryCache : ReadOnlyBinaryCache { - BinaryCache(RemoveFilesystem& fs); - BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info); - BinaryCache(BinaryProviders&& providers, RemoveFilesystem& fs, const ToolCache& tools); + static ExpectedL make(const VcpkgCmdArguments& args, const VcpkgPaths& paths, MessageSink& sink); + + BinaryCache(Filesystem& fs); + BinaryCache(const BinaryCache&) = delete; + BinaryCache(BinaryCache&&) = default; ~BinaryCache(); /// Called upon a successful build of `action` to store those contents in the binary cache. void push_success(const InstallPlanAction& action); private: - RemoveFilesystem& m_fs; + BinaryCache(BinaryProviders&& providers, Filesystem& fs); + + Filesystem& m_fs; Optional m_zip_tool; bool m_needs_nuspec_data = false; bool m_needs_zip_file = false; diff --git a/src/vcpkg-test/binarycaching.cpp b/src/vcpkg-test/binarycaching.cpp index 5cadcc3e5d..b74d930c5b 100644 --- a/src/vcpkg-test/binarycaching.cpp +++ b/src/vcpkg-test/binarycaching.cpp @@ -37,7 +37,10 @@ struct KnowNothingBinaryProvider : IReadBinaryProvider } } - StringView vendor() const override { return "KnowNothing"; } + LocalizedString restored_message(size_t, std::chrono::high_resolution_clock::duration) const override + { + return LocalizedString::from_raw("Nothing"); + } }; TEST_CASE ("CacheStatus operations", "[BinaryCache]") diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index 1c7aef4c46..4e84dc7ed3 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -316,15 +316,17 @@ namespace vcpkg #endif } - ZipTool::ZipTool(RemoveFilesystem& fs, const ToolCache& cache, MessageSink& status_sink) - : fs(&fs) + ExpectedL ZipTool::make(Filesystem& fs, const ToolCache& cache, MessageSink& status_sink) + { + ZipTool ret; + ret.fs = &fs; #if defined(_WIN32) - , seven_zip(cache.get_tool_path(Tools::SEVEN_ZIP, status_sink)) + ret.seven_zip = cache.get_tool_path(Tools::SEVEN_ZIP, status_sink); #endif - { // Unused on non-Windows (void)cache; (void)status_sink; + return std::move(ret); } Command ZipTool::decompress_zip_archive_cmd(const Path& dst, const Path& archive_path) const diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index f76368c26b..822627689d 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -174,7 +174,7 @@ namespace if (last - first >= 4 && is_slash(first[3]) && (last - first == 4 || !is_slash(first[4])) // \xx\$ && ((is_slash(first[1]) && (first[2] == '?' || first[2] == '.')) // \\?\$ or \\.\$ || (first[1] == '?' && first[2] == '?'))) - { // \??\$ + { // \??\$ return first + 3; } @@ -260,7 +260,7 @@ namespace } if (*extension == '.') - { // we might have found the end of stem + { // we might have found the end of stem if (filename == extension - 1 && extension[-1] == '.') { // dotdot special case return ads; @@ -293,9 +293,9 @@ namespace #if defined(_WIN32) const auto ads = std::find(filename, last, ':'); // strip alternate data streams in intra-filename decomposition const auto extension = find_extension(filename, ads); -#else // ^^^ _WIN32 / !_WIN32 vvv +#else // ^^^ _WIN32 / !_WIN32 vvv const auto extension = find_extension(filename, last); -#endif // _WIN32 +#endif // _WIN32 return StringView(filename, static_cast(extension - filename)); } @@ -1535,6 +1535,8 @@ namespace vcpkg int WriteFilePointer::put(int c) const noexcept { return ::fputc(c, m_fs); } + ILineReader::~ILineReader() = default; + std::string Filesystem::read_contents(const Path& file_path, LineInfo li) const { std::error_code ec; @@ -1714,7 +1716,7 @@ namespace vcpkg } } - bool RemoveFilesystem::remove(const Path& target, LineInfo li) + bool Filesystem::remove(const Path& target, LineInfo li) { std::error_code ec; auto r = this->remove(target, ec); @@ -1906,7 +1908,7 @@ namespace vcpkg } } - void RemoveFilesystem::remove_all(const Path& base, LineInfo li) + void Filesystem::remove_all(const Path& base, LineInfo li) { std::error_code ec; Path failure_point; @@ -1923,7 +1925,7 @@ namespace vcpkg } } - void RemoveFilesystem::remove_all(const Path& base, std::error_code& ec) + void Filesystem::remove_all(const Path& base, std::error_code& ec) { Path failure_point; this->remove_all(base, ec, failure_point); @@ -2878,7 +2880,7 @@ namespace vcpkg StatsTimer t(g_us_filesystem_stats); #if defined(_WIN32) return stdfs::create_directories(to_stdfs_path(new_directory), ec); -#else // ^^^ _WIN32 // !_WIN32 vvv +#else // ^^^ _WIN32 // !_WIN32 vvv ec.clear(); if (new_directory.empty()) { diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 7a76ef9f34..5e1a39311d 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -217,23 +217,16 @@ namespace size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override { - if (!request.zip_path) return 0; const auto& zip_path = request.zip_path.value_or_exit(VCPKG_LINE_INFO); const auto archive_subpath = files_archive_subpath(request.package_abi); - int count_stored = 0; + size_t count_stored = 0; for (const auto& archives_root_dir : m_dirs) { const auto archive_path = archives_root_dir / archive_subpath; std::error_code ec; + m_fs.create_directories(archive_path.parent_path(), IgnoreErrors{}); m_fs.copy_file(zip_path, archive_path, CopyOptions::overwrite_existing, ec); - if (ec) - { - ec.clear(); - m_fs.create_directories(archive_path.parent_path(), IgnoreErrors{}); - m_fs.copy_file(zip_path, archive_path, CopyOptions::overwrite_existing, ec); - } - if (ec) { msg_sink.println(Color::warning, @@ -257,25 +250,24 @@ namespace std::vector m_dirs; }; - enum class ToRemove + enum class RemoveWhen { nothing, - remove_always, - remove_on_fail, + always, + on_fail, }; struct ZipResource { - ZipResource(Path&& p, ToRemove t) : path(std::move(p)), to_remove(t) { } + ZipResource(Path&& p, RemoveWhen t) : path(std::move(p)), to_remove(t) { } Path path; - ToRemove to_remove; + RemoveWhen to_remove; }; // This middleware class contains logic for BinaryProviders that operate on zip files. // Derived classes must implement: // - acquire_zips() - // - post_decompress() // - IReadBinaryProvider::precheck() struct ZipReadBinaryProvider : IReadBinaryProvider { @@ -316,22 +308,20 @@ namespace post_decompress(zip_path, job_results[j].has_value()); } - size_t num_restored = std::count(out_status.begin(), out_status.end(), RestoreResult::restored); - msg::println(msgRestoredPackagesFromVendor, - msg::count = num_restored, - msg::elapsed = timer.elapsed(), - msg::value = vendor()); } void post_decompress(const ZipResource& r, bool succeeded) const { - if ((!succeeded && r.to_remove == ToRemove::remove_on_fail) || r.to_remove == ToRemove::remove_always) + if ((!succeeded && r.to_remove == RemoveWhen::on_fail) || r.to_remove == RemoveWhen::always) { m_fs.remove(r.path, IgnoreErrors{}); } } - // Download/find the zip files corresponding with a set of actions + // For every action denoted by actions, at corresponding indicies in out_zips, stores a ZipResource indicating + // the downloaded location. + // + // Leaving an Optional disengaged indicates that the cache does not contain the requested zip. virtual void acquire_zips(View actions, Span> out_zips) const = 0; @@ -357,8 +347,8 @@ namespace if (m_fs.exists(archive_path, IgnoreErrors{})) { auto to_remove = actions[i]->build_options.purge_decompress_failure == PurgeDecompressFailure::YES - ? ToRemove::remove_on_fail - : ToRemove::nothing; + ? RemoveWhen::on_fail + : RemoveWhen::nothing; out_zip_paths[i].emplace(std::move(archive_path), to_remove); } } @@ -380,8 +370,14 @@ namespace cache_status[idx] = any_available ? CacheAvailability::available : CacheAvailability::unavailable; } } - - StringView vendor() const override { return m_dir; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromFiles, + msg::count = count, + msg::elapsed = ElapsedTime(elapsed), + msg::path = m_dir); + } private: Path m_dir; @@ -443,8 +439,8 @@ namespace for (size_t idx = 0; idx < actions.size(); ++idx) { auto&& action = *actions[idx]; - auto uri = m_url_template.instantiate_variables(BinaryPackageReadInfo{action}); - url_paths.emplace_back(std::move(uri), make_temp_archive_path(m_buildtrees, action.spec)); + url_paths.emplace_back(m_url_template.instantiate_variables(BinaryPackageReadInfo{action}), + make_temp_archive_path(m_buildtrees, action.spec)); } auto codes = download_files(m_fs, url_paths, m_url_template.headers); @@ -453,7 +449,7 @@ namespace { if (codes[i] == 200) { - out_zip_paths[i].emplace(std::move(url_paths[i].second), ToRemove::remove_always); + out_zip_paths[i].emplace(std::move(url_paths[i].second), RemoveWhen::always); } } } @@ -473,7 +469,11 @@ namespace } } - StringView vendor() const override { return "HTTP servers"; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromHTTP, msg::count = count, msg::elapsed = ElapsedTime(elapsed)); + } Path m_buildtrees; UrlTemplate m_url_template; @@ -682,7 +682,11 @@ namespace // Prechecking is too expensive with NuGet, so it is not implemented void precheck(View, Span) const override { } - StringView vendor() const override { return "NuGet"; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromNuGet, msg::count = count, msg::elapsed = ElapsedTime(elapsed)); + } void fetch(View actions, Span out_status) const override { @@ -857,14 +861,18 @@ namespace { if (codes[i] == 200) { - out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), ToRemove::remove_always); + out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), RemoveWhen::always); } } } void precheck(View, Span) const override { } - StringView vendor() const override { return "GitHub Actions Cache"; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromGHA, msg::count = count, msg::elapsed = ElapsedTime(elapsed)); + } static constexpr StringLiteral m_accept_header = "Accept: application/json;api-version=6.0-preview.1"; @@ -963,7 +971,8 @@ namespace { virtual ~IObjectStorageTool() = default; - virtual StringLiteral vendor() const = 0; + virtual LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const = 0; virtual ExpectedL stat(StringView url) const = 0; virtual ExpectedL download_file(StringView object, const Path& archive) const = 0; virtual ExpectedL upload_file(StringView object, const Path& archive) const = 0; @@ -999,7 +1008,7 @@ namespace auto res = m_tool->download_file(make_object_path(m_prefix, abi), tmp); if (res) { - out_zip_paths[idx].emplace(std::move(tmp), ToRemove::remove_always); + out_zip_paths[idx].emplace(std::move(tmp), RemoveWhen::always); } else { @@ -1025,7 +1034,11 @@ namespace } } - StringView vendor() const override { return m_tool->vendor(); } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return m_tool->restored_message(count, elapsed); + } Path m_buildtrees; std::string m_prefix; @@ -1074,7 +1087,11 @@ namespace { GcsStorageTool(const ToolCache& cache, MessageSink& sink) : m_tool(cache.get_tool_path(Tools::GSUTIL, sink)) { } - StringLiteral vendor() const override { return "GCS"; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromGCS, msg::count = count, msg::elapsed = ElapsedTime(elapsed)); + } Command command() const { return m_tool; } @@ -1106,7 +1123,11 @@ namespace { } - StringLiteral vendor() const override { return "AWS"; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromAWS, msg::count = count, msg::elapsed = ElapsedTime(elapsed)); + } Command command() const { return m_tool; } @@ -1153,7 +1174,11 @@ namespace { CosStorageTool(const ToolCache& cache, MessageSink& sink) : m_tool(cache.get_tool_path(Tools::COSCLI, sink)) { } - StringLiteral vendor() const override { return "COS"; } + LocalizedString restored_message(size_t count, + std::chrono::high_resolution_clock::duration elapsed) const override + { + return msg::format(msgRestoredPackagesFromCOS, msg::count = count, msg::elapsed = ElapsedTime(elapsed)); + } Command command() const { return m_tool; } @@ -1878,106 +1903,128 @@ namespace vcpkg ret.nuget_prefix = s.nuget_prefix; - Optional lazy_zip_tool; - auto zip_tool = [&]() -> const ZipTool& { - if (!lazy_zip_tool) lazy_zip_tool.emplace(fs, tools, stdout_sink); - return *lazy_zip_tool.get(); - }; + std::shared_ptr gcs_tool; + if (!s.gcs_read_prefixes.empty() || !s.gcs_write_prefixes.empty()) + { + gcs_tool = std::make_shared(tools, stdout_sink); + } + std::shared_ptr aws_tool; + if (!s.aws_read_prefixes.empty() || !s.aws_write_prefixes.empty()) + { + aws_tool = std::make_shared(tools, stdout_sink, s.aws_no_sign_request); + } + std::shared_ptr cos_tool; + if (!s.cos_read_prefixes.empty() || !s.cos_write_prefixes.empty()) + { + cos_tool = std::make_shared(tools, stdout_sink); + } - std::shared_ptr lazy_gcs_tool; - auto gcs_tool = [&]() -> const std::shared_ptr& { - if (!lazy_gcs_tool) lazy_gcs_tool = std::make_shared(tools, stdout_sink); - return lazy_gcs_tool; - }; + if (s.gha_read || s.gha_write) + { + if (!args.actions_cache_url.has_value() || !args.actions_runtime_token.has_value()) + return msg::format_error(msgGHAParametersMissing, + msg::url = + "https://learn.microsoft.com/en-us/vcpkg/users/binarycaching#gha"); + } - std::shared_ptr lazy_aws_tool; - auto aws_tool = [&]() -> const std::shared_ptr& { - if (!lazy_aws_tool) - lazy_aws_tool = std::make_shared(tools, stdout_sink, s.aws_no_sign_request); - return lazy_aws_tool; - }; + if (!s.archives_to_read.empty() || !s.url_templates_to_get.empty() || !s.gcs_read_prefixes.empty() || + !s.aws_read_prefixes.empty() || !s.cos_read_prefixes.empty() || s.gha_read) + { + auto maybe_zip_tool = ZipTool::make(fs, tools, stdout_sink); + if (!maybe_zip_tool.has_value()) + { + return std::move(maybe_zip_tool).error(); + } + const auto& zip_tool = *maybe_zip_tool.get(); - std::shared_ptr lazy_cos_tool; - auto cos_tool = [&]() -> const std::shared_ptr& { - if (!lazy_cos_tool) lazy_cos_tool = std::make_shared(tools, stdout_sink); - return lazy_cos_tool; - }; - Optional lazy_nuget_base; - auto nuget_base = [&]() -> const NugetBaseBinaryProvider& { - if (!lazy_nuget_base) - lazy_nuget_base.emplace( - fs, NuGetTool(tools, stdout_sink, s), paths.packages(), buildtrees, s.nuget_prefix); - return *lazy_nuget_base.get(); - }; + for (auto&& dir : s.archives_to_read) + { + ret.read.push_back(std::make_unique(zip_tool, fs, std::move(dir))); + } + + for (auto&& url : s.url_templates_to_get) + { + ret.read.push_back( + std::make_unique(zip_tool, fs, buildtrees, std::move(url), s.secrets)); + } - for (auto&& dir : s.archives_to_read) - ret.read.push_back(std::make_unique(zip_tool(), fs, std::move(dir))); + for (auto&& prefix : s.gcs_read_prefixes) + { + ret.read.push_back( + std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), gcs_tool)); + } + + for (auto&& prefix : s.aws_read_prefixes) + { + ret.read.push_back( + std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), aws_tool)); + } + + for (auto&& prefix : s.cos_read_prefixes) + { + ret.read.push_back( + std::make_unique(zip_tool, fs, buildtrees, std::move(prefix), cos_tool)); + } + + if (s.gha_read) + { + const auto& url = *args.actions_cache_url.get(); + const auto& token = *args.actions_runtime_token.get(); + ret.read.push_back(std::make_unique(zip_tool, fs, buildtrees, url, token)); + } + } if (!s.archives_to_write.empty()) + { ret.write.push_back(std::make_unique(fs, std::move(s.archives_to_write))); - - for (auto&& url : s.url_templates_to_get) - ret.read.push_back( - std::make_unique(zip_tool(), fs, buildtrees, std::move(url), s.secrets)); + } if (!s.url_templates_to_put.empty()) + { ret.write.push_back( std::make_unique(fs, std::move(s.url_templates_to_put), s.secrets)); - - for (auto&& prefix : s.gcs_read_prefixes) - ret.read.push_back( - std::make_unique(zip_tool(), fs, buildtrees, std::move(prefix), gcs_tool())); + } if (!s.gcs_write_prefixes.empty()) + { ret.write.push_back( - std::make_unique(std::move(s.gcs_write_prefixes), gcs_tool())); - - for (auto&& prefix : s.aws_read_prefixes) - ret.read.push_back( - std::make_unique(zip_tool(), fs, buildtrees, std::move(prefix), aws_tool())); + std::make_unique(std::move(s.gcs_write_prefixes), gcs_tool)); + } if (!s.aws_write_prefixes.empty()) + { ret.write.push_back( - std::make_unique(std::move(s.aws_write_prefixes), aws_tool())); - - for (auto&& prefix : s.cos_read_prefixes) - ret.read.push_back( - std::make_unique(zip_tool(), fs, buildtrees, std::move(prefix), cos_tool())); + std::make_unique(std::move(s.aws_write_prefixes), aws_tool)); + } if (!s.cos_write_prefixes.empty()) + { ret.write.push_back( - std::make_unique(std::move(s.cos_write_prefixes), cos_tool())); - - if (s.gha_read || s.gha_write) + std::make_unique(std::move(s.cos_write_prefixes), cos_tool)); + } + if (s.gha_write) { - if (!args.actions_cache_url.has_value() || !args.actions_runtime_token.has_value()) - return msg::format_error(msgGHAParametersMissing, - msg::url = - "https://learn.microsoft.com/en-us/vcpkg/users/binarycaching#gha"); const auto& url = *args.actions_cache_url.get(); const auto& token = *args.actions_runtime_token.get(); - if (s.gha_read) - ret.read.push_back(std::make_unique(zip_tool(), fs, buildtrees, url, token)); - if (s.gha_write) ret.write.push_back(std::make_unique(fs, url, token)); + ret.write.push_back(std::make_unique(fs, url, token)); } - if (!s.sources_to_read.empty()) - ret.read.push_back( - std::make_unique(nuget_base(), nuget_sources_arg(s.sources_to_read))); - for (auto&& config : s.configs_to_read) - ret.read.push_back( - std::make_unique(nuget_base(), nuget_configfile_arg(config))); - if (!s.sources_to_write.empty() || !s.configs_to_write.empty()) + if (!s.sources_to_read.empty() || !s.configs_to_read.empty() || !s.sources_to_write.empty() || + !s.configs_to_write.empty()) { - ret.write.push_back(std::make_unique( - nuget_base(), std::move(s.sources_to_write), std::move(s.configs_to_write))); + NugetBaseBinaryProvider nuget_base( + fs, NuGetTool(tools, stdout_sink, s), paths.packages(), buildtrees, s.nuget_prefix); + if (!s.sources_to_read.empty()) + ret.read.push_back( + std::make_unique(nuget_base, nuget_sources_arg(s.sources_to_read))); + for (auto&& config : s.configs_to_read) + ret.read.push_back( + std::make_unique(nuget_base, nuget_configfile_arg(config))); + if (!s.sources_to_write.empty() || !s.configs_to_write.empty()) + { + ret.write.push_back(std::make_unique( + nuget_base, std::move(s.sources_to_write), std::move(s.configs_to_write))); + } } } return std::move(ret); } - ReadOnlyBinaryCache::ReadOnlyBinaryCache(const VcpkgCmdArguments& args, - const VcpkgPaths& paths, - const LineInfo& info) - : ReadOnlyBinaryCache(make_binary_providers(args, paths).value_or_exit(info)) - { - } - ReadOnlyBinaryCache::ReadOnlyBinaryCache(BinaryProviders&& providers) : m_config(std::move(providers)) { } void ReadOnlyBinaryCache::fetch(View actions) @@ -2020,10 +2067,8 @@ namespace vcpkg ++num_restored; } } - msg::println(msgRestoredPackagesFromVendor, - msg::count = num_restored, - msg::elapsed = timer.elapsed(), - msg::value = provider->vendor()); + msg::println(provider->restored_message( + num_restored, timer.elapsed().as())); } } @@ -2067,13 +2112,14 @@ namespace vcpkg for (size_t i = 0; i < action_ptrs.size(); ++i) { + auto&& this_status = m_status[*action_ptrs[i]->package_abi().get()]; if (cache_result[i] == CacheAvailability::available) { - m_status[*action_ptrs[i]->package_abi().get()].mark_available(provider.get()); + this_status.mark_available(provider.get()); } else if (cache_result[i] == CacheAvailability::unavailable) { - m_status[*action_ptrs[i]->package_abi().get()].mark_unavailable(provider.get()); + this_status.mark_unavailable(provider.get()); } } } @@ -2083,20 +2129,33 @@ namespace vcpkg }); } - BinaryCache::BinaryCache(RemoveFilesystem& fs) : m_fs(fs) { } + BinaryCache::BinaryCache(Filesystem& fs) : m_fs(fs) { } - BinaryCache::BinaryCache(const VcpkgCmdArguments& args, const VcpkgPaths& paths, const LineInfo& info) - : BinaryCache( - make_binary_providers(args, paths).value_or_exit(info), paths.get_filesystem(), paths.get_tool_cache()) + ExpectedL BinaryCache::make(const VcpkgCmdArguments& args, const VcpkgPaths& paths, MessageSink& sink) { + return make_binary_providers(args, paths).then([&](BinaryProviders&& p) -> ExpectedL { + BinaryCache b(std::move(p), paths.get_filesystem()); + b.m_needs_nuspec_data = Util::any_of(b.m_config.write, [](auto&& p) { return p->needs_nuspec_data(); }); + b.m_needs_zip_file = Util::any_of(b.m_config.write, [](auto&& p) { return p->needs_zip_file(); }); + if (b.m_needs_zip_file) + { + auto maybe_zt = ZipTool::make(paths.get_filesystem(), paths.get_tool_cache(), sink); + if (auto z = maybe_zt.get()) + { + b.m_zip_tool.emplace(std::move(*z)); + } + else + { + return std::move(maybe_zt).error(); + } + } + return std::move(b); + }); } - BinaryCache::BinaryCache(BinaryProviders&& providers, RemoveFilesystem& fs, const ToolCache& tools) + BinaryCache::BinaryCache(BinaryProviders&& providers, Filesystem& fs) : ReadOnlyBinaryCache(std::move(providers)), m_fs(fs) { - m_needs_nuspec_data = Util::any_of(m_config.write, [](auto&& p) { return p->needs_nuspec_data(); }); - m_needs_zip_file = Util::any_of(m_config.write, [](auto&& p) { return p->needs_zip_file(); }); - if (m_needs_zip_file) m_zip_tool.emplace(fs, tools, stdout_sink); } BinaryCache::~BinaryCache() { } @@ -2141,7 +2200,10 @@ namespace vcpkg size_t num_destinations = 0; for (auto&& provider : m_config.write) { - num_destinations += provider->push_success(request, stdout_sink); + if (!provider->needs_zip_file() || request.zip_path.has_value()) + { + num_destinations += provider->push_success(request, stdout_sink); + } } if (request.zip_path) { diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 6cc3c047b4..984fe327b6 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -138,7 +138,7 @@ namespace vcpkg::Build action->build_options.clean_buildtrees = CleanBuildtrees::NO; action->build_options.clean_packages = CleanPackages::NO; - BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); + auto binary_cache = BinaryCache::make(args, paths, stdout_sink).value_or_exit(VCPKG_LINE_INFO); const ElapsedTimer build_timer; const auto result = build_package(args, paths, *action, build_logs_recorder, status_db); msg::print(msgElapsedForPackage, msg::spec = spec, msg::elapsed = build_timer); diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 6611d1829a..ec897f6ee7 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -432,7 +432,7 @@ namespace vcpkg::Commands::CI } auto action_plan = compute_full_plan(paths, provider, var_provider, all_default_full_specs, serialize_options); - BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); + auto binary_cache = BinaryCache::make(args, paths, stdout_sink).value_or_exit(VCPKG_LINE_INFO); const auto precheck_results = binary_cache.precheck(action_plan.install_actions); auto split_specs = compute_action_statuses(ExclusionPredicate{&exclusions_map}, var_provider, precheck_results, action_plan); diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index b851926e9c..416ed40433 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -1271,8 +1271,8 @@ namespace vcpkg track_install_plan(action_plan); - BinaryCache binary_cache = - only_downloads ? BinaryCache(paths.get_filesystem()) : BinaryCache(args, paths, VCPKG_LINE_INFO); + auto binary_cache = only_downloads ? BinaryCache(paths.get_filesystem()) + : BinaryCache::make(args, paths, stdout_sink).value_or_exit(VCPKG_LINE_INFO); binary_cache.fetch(action_plan.install_actions); const InstallSummary summary = Install::execute_plan( args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder()); diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 9e13235ac3..8b2c35f673 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -136,7 +136,8 @@ namespace vcpkg::Commands::SetInstalled track_install_plan(action_plan); - BinaryCache binary_cache = only_downloads ? BinaryCache(paths.get_filesystem()) : BinaryCache(args, paths, VCPKG_LINE_INFO); + auto binary_cache = only_downloads ? BinaryCache(paths.get_filesystem()) + : BinaryCache::make(args, paths, stdout_sink).value_or_exit(VCPKG_LINE_INFO); binary_cache.fetch(action_plan.install_actions); const auto summary = Install::execute_plan( args, action_plan, keep_going, paths, status_db, binary_cache, null_build_logs_recorder()); diff --git a/src/vcpkg/commands.upgrade.cpp b/src/vcpkg/commands.upgrade.cpp index 9aed0d400f..139edb0a8e 100644 --- a/src/vcpkg/commands.upgrade.cpp +++ b/src/vcpkg/commands.upgrade.cpp @@ -205,7 +205,7 @@ namespace vcpkg::Commands::Upgrade var_provider.load_tag_vars(action_plan, provider, host_triplet); - BinaryCache binary_cache(args, paths, VCPKG_LINE_INFO); + auto binary_cache = BinaryCache::make(args, paths, stdout_sink).value_or_exit(VCPKG_LINE_INFO); compute_all_abis(paths, action_plan, var_provider, status_db); binary_cache.fetch(action_plan.install_actions); const InstallSummary summary = Install::execute_plan( diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index b77938f118..b8e8395e23 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -615,7 +615,8 @@ namespace vcpkg::Export::Prefab Debug::print( fmt::format("Exporting AAR and POM\n\tAAR path {}\n\tPOM path {}", exported_archive_path, pom_path)); - ZipTool zip(paths.get_filesystem(), paths.get_tool_cache(), stdout_sink); + auto zip = ZipTool::make(paths.get_filesystem(), paths.get_tool_cache(), stdout_sink) + .value_or_exit(VCPKG_LINE_INFO); auto compress_result = zip.compress_directory_to_zip(package_directory, exported_archive_path); if (!compress_result) From 07d92a6424a25bb4b148cc66b365f10eb55fd043 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 9 May 2023 21:40:19 +0000 Subject: [PATCH 7/9] Additional PR comments --- include/vcpkg/base/fwd/files.h | 1 - src/vcpkg/binarycaching.cpp | 27 ++++++++++----------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/include/vcpkg/base/fwd/files.h b/include/vcpkg/base/fwd/files.h index 0c8d1e04dd..dd2be72618 100644 --- a/include/vcpkg/base/fwd/files.h +++ b/include/vcpkg/base/fwd/files.h @@ -48,7 +48,6 @@ namespace vcpkg struct IExclusiveFileLock; struct ILineReader; struct FileContents; - struct RemoveFilesystem; struct Filesystem; struct NotExtensionCaseSensitive; struct NotExtensionCaseInsensitive; diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 5e1a39311d..3cf0a22b16 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -733,13 +733,6 @@ namespace size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override { - if (!request.nuspec.has_value()) - { - Checks::unreachable( - VCPKG_LINE_INFO, - "request.info.nuspec must be non empty because needs_nuspec_data() should return true"); - } - auto& spec = request.spec; auto nuspec_path = m_buildtrees / spec.name() / spec.triplet().canonical_name() + ".nuspec"; @@ -984,11 +977,11 @@ namespace Filesystem& fs, const Path& buildtrees, std::string&& prefix, - std::shared_ptr tool) + const std::shared_ptr& tool) : ZipReadBinaryProvider(std::move(zip), fs) , m_buildtrees(buildtrees) , m_prefix(std::move(prefix)) - , m_tool(std::move(tool)) + , m_tool(tool) { } @@ -2161,13 +2154,13 @@ namespace vcpkg void BinaryCache::push_success(const InstallPlanAction& action) { - if (action.package_abi().has_value()) + if (auto abi = action.package_abi().get()) { - bool restored = m_status[*action.package_abi().get()].is_restored(); + bool restored = m_status[*abi].is_restored(); // Purge all status information on push_success (cache invalidation) // - push_success may delete packages/ (invalidate restore) // - push_success may make the package available from providers (invalidate unavailable) - m_status.erase(*action.package_abi().get()); + m_status.erase(*abi); if (!restored && !m_config.write.empty()) { ElapsedTimer timer; @@ -2183,7 +2176,11 @@ namespace vcpkg Path zip_path = request.package_dir + ".zip"; auto compress_result = m_zip_tool.value_or_exit(VCPKG_LINE_INFO) .compress_directory_to_zip(request.package_dir, zip_path); - if (!compress_result) + if (compress_result) + { + request.zip_path = std::move(zip_path); + } + else { stdout_sink.println( Color::warning, @@ -2191,10 +2188,6 @@ namespace vcpkg .append_raw(' ') .append_raw(compress_result.error())); } - else - { - request.zip_path = std::move(zip_path); - } } size_t num_destinations = 0; From 12919d3d8e8dd742dd4ddccfa99acb1eb7830f63 Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Wed, 10 May 2023 00:06:43 +0000 Subject: [PATCH 8/9] Address PR comments --- include/vcpkg/archives.h | 5 ++--- src/vcpkg/archives.cpp | 9 +++++---- src/vcpkg/binarycaching.cpp | 8 ++++---- src/vcpkg/commands.set-installed.cpp | 4 ---- src/vcpkg/export.prefab.cpp | 6 +++--- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/include/vcpkg/archives.h b/include/vcpkg/archives.h index c3b15706af..3acc75e82c 100644 --- a/include/vcpkg/archives.h +++ b/include/vcpkg/archives.h @@ -32,19 +32,18 @@ namespace vcpkg struct ZipTool { - static ExpectedL make(Filesystem& fs, const ToolCache& tools, MessageSink& status_sink); + static ExpectedL make(const ToolCache& tools, MessageSink& status_sink); private: ZipTool() = default; - Filesystem* fs = nullptr; #if defined _WIN32 Path seven_zip; #endif public: // Compress the source directory into the destination file. - ExpectedL compress_directory_to_zip(const Path& source, const Path& destination) const; + ExpectedL compress_directory_to_zip(Filesystem& fs, const Path& source, const Path& destination) const; Command decompress_zip_archive_cmd(const Path& dst, const Path& archive_path) const; }; diff --git a/src/vcpkg/archives.cpp b/src/vcpkg/archives.cpp index 4e84dc7ed3..b94e7251c8 100644 --- a/src/vcpkg/archives.cpp +++ b/src/vcpkg/archives.cpp @@ -293,9 +293,11 @@ namespace vcpkg fs.rename_with_retry(to_path_partial, to_path, VCPKG_LINE_INFO); } - ExpectedL ZipTool::compress_directory_to_zip(const Path& source, const Path& destination) const + ExpectedL ZipTool::compress_directory_to_zip(Filesystem& fs, + const Path& source, + const Path& destination) const { - fs->remove(destination, VCPKG_LINE_INFO); + fs.remove(destination, VCPKG_LINE_INFO); #if defined(_WIN32) return flatten(cmd_execute_and_capture_output( Command{seven_zip}.string_arg("a").string_arg(destination).string_arg(source / "*"), @@ -316,10 +318,9 @@ namespace vcpkg #endif } - ExpectedL ZipTool::make(Filesystem& fs, const ToolCache& cache, MessageSink& status_sink) + ExpectedL ZipTool::make(const ToolCache& cache, MessageSink& status_sink) { ZipTool ret; - ret.fs = &fs; #if defined(_WIN32) ret.seven_zip = cache.get_tool_path(Tools::SEVEN_ZIP, status_sink); #endif diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 3cf0a22b16..85a7e82ad5 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -752,7 +752,7 @@ namespace return 0; } - int count_stored = 0; + size_t count_stored = 0; auto nupkg_path = m_buildtrees / make_nugetref(request, m_nuget_prefix).nupkg_filename(); for (auto&& write_src : m_sources) { @@ -1923,7 +1923,7 @@ namespace vcpkg if (!s.archives_to_read.empty() || !s.url_templates_to_get.empty() || !s.gcs_read_prefixes.empty() || !s.aws_read_prefixes.empty() || !s.cos_read_prefixes.empty() || s.gha_read) { - auto maybe_zip_tool = ZipTool::make(fs, tools, stdout_sink); + auto maybe_zip_tool = ZipTool::make(tools, stdout_sink); if (!maybe_zip_tool.has_value()) { return std::move(maybe_zip_tool).error(); @@ -2132,7 +2132,7 @@ namespace vcpkg b.m_needs_zip_file = Util::any_of(b.m_config.write, [](auto&& p) { return p->needs_zip_file(); }); if (b.m_needs_zip_file) { - auto maybe_zt = ZipTool::make(paths.get_filesystem(), paths.get_tool_cache(), sink); + auto maybe_zt = ZipTool::make(paths.get_tool_cache(), sink); if (auto z = maybe_zt.get()) { b.m_zip_tool.emplace(std::move(*z)); @@ -2175,7 +2175,7 @@ namespace vcpkg { Path zip_path = request.package_dir + ".zip"; auto compress_result = m_zip_tool.value_or_exit(VCPKG_LINE_INFO) - .compress_directory_to_zip(request.package_dir, zip_path); + .compress_directory_to_zip(m_fs, request.package_dir, zip_path); if (compress_result) { request.zip_path = std::move(zip_path); diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 8b2c35f673..a41ac36751 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -122,10 +122,6 @@ namespace vcpkg::Commands::SetInstalled fs.write_contents(pkgsconfig_path, pkgsconfig_contents, VCPKG_LINE_INFO); msg::println(msgWroteNuGetPkgConfInfo, msg::path = pkgsconfig_path); } - else if (dry_run != DryRun::Yes) - { - compute_all_abis(paths, action_plan, cmake_vars, status_db); - } if (dry_run == DryRun::Yes) { diff --git a/src/vcpkg/export.prefab.cpp b/src/vcpkg/export.prefab.cpp index b8e8395e23..bfd4c5e6c1 100644 --- a/src/vcpkg/export.prefab.cpp +++ b/src/vcpkg/export.prefab.cpp @@ -615,10 +615,10 @@ namespace vcpkg::Export::Prefab Debug::print( fmt::format("Exporting AAR and POM\n\tAAR path {}\n\tPOM path {}", exported_archive_path, pom_path)); - auto zip = ZipTool::make(paths.get_filesystem(), paths.get_tool_cache(), stdout_sink) - .value_or_exit(VCPKG_LINE_INFO); + auto zip = ZipTool::make(paths.get_tool_cache(), stdout_sink).value_or_exit(VCPKG_LINE_INFO); - auto compress_result = zip.compress_directory_to_zip(package_directory, exported_archive_path); + auto compress_result = + zip.compress_directory_to_zip(paths.get_filesystem(), package_directory, exported_archive_path); if (!compress_result) { Checks::msg_exit_with_message( From c9b5a57f864b0e4b926e40d5140e5aa9ed18bdbb Mon Sep 17 00:00:00 2001 From: Robert Schumacher Date: Tue, 9 May 2023 21:46:22 +0000 Subject: [PATCH 9/9] Format and regenerate messages --- locales/messages.json | 19 +++++++++++++++---- src/vcpkg/base/files.cpp | 10 +++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/locales/messages.json b/locales/messages.json index 24bc97bcf1..564289e000 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -168,6 +168,7 @@ "_BaselineMissing.comment": "An example of {package_name} is zlib. An example of {version} is 1.3.8.", "BaselineMissingDefault": "The baseline.json from commit `\"{commit_sha}\"` in the repo {url} was invalid (did not contain a \"default\" field).", "_BaselineMissingDefault.comment": "An example of {commit_sha} is 7cfad47ae9f68b183983090afd6337cd60fd4949. An example of {url} is https://github.com/microsoft/vcpkg.", + "BinaryCacheVendorHTTP": "HTTP servers", "BinarySourcesArg": "Add sources for binary caching. See 'vcpkg help binarycaching'.", "BinaryWithInvalidArchitecture": "{path}\n Expected: {expected}, but was {actual}", "_BinaryWithInvalidArchitecture.comment": "{expected} and {actual} are architectures An example of {path} is /foo/bar.", @@ -1239,10 +1240,20 @@ "_RemovingPackage.comment": "An example of {action_index} is 340. An example of {count} is 42. An example of {spec} is zlib:x64-windows.", "ResponseFileCode": "@response_file", "_ResponseFileCode.comment": "Explains to the user that they can use response files on the command line, 'response_file' must have no spaces and be a legal file name.", - "RestoredPackage": "Restored package from \"{path}\"", - "_RestoredPackage.comment": "An example of {path} is /foo/bar.", - "RestoredPackagesFromVendor": "Restored {count} package(s) from {value} in {elapsed}. Use --debug to see more details.", - "_RestoredPackagesFromVendor.comment": "{value} may be either a 'vendor' like 'Azure' or 'NuGet', or a file path like C:\\example or /usr/example An example of {count} is 42. An example of {elapsed} is 3.532 min.", + "RestoredPackagesFromAWS": "Restored {count} package(s) from AWS in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromAWS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", + "RestoredPackagesFromCOS": "Restored {count} package(s) from COS in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromCOS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", + "RestoredPackagesFromFiles": "Restored {count} package(s) from {path} in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromFiles.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min. An example of {path} is /foo/bar.", + "RestoredPackagesFromGCS": "Restored {count} package(s) from GCS in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromGCS.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", + "RestoredPackagesFromGHA": "Restored {count} package(s) from GitHub Actions Cache in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromGHA.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", + "RestoredPackagesFromHTTP": "Restored {count} package(s) from HTTP servers in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromHTTP.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", + "RestoredPackagesFromNuGet": "Restored {count} package(s) from NuGet in {elapsed}. Use --debug to see more details.", + "_RestoredPackagesFromNuGet.comment": "An example of {count} is 42. An example of {elapsed} is 3.532 min.", "ResultsHeader": "RESULTS", "_ResultsHeader.comment": "Displayed before a list of installation results.", "ScriptAssetCacheRequiresScript": "expected arguments: asset config 'x-script' requires exactly the exec template as an argument", diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index 822627689d..244232fa13 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -174,7 +174,7 @@ namespace if (last - first >= 4 && is_slash(first[3]) && (last - first == 4 || !is_slash(first[4])) // \xx\$ && ((is_slash(first[1]) && (first[2] == '?' || first[2] == '.')) // \\?\$ or \\.\$ || (first[1] == '?' && first[2] == '?'))) - { // \??\$ + { // \??\$ return first + 3; } @@ -260,7 +260,7 @@ namespace } if (*extension == '.') - { // we might have found the end of stem + { // we might have found the end of stem if (filename == extension - 1 && extension[-1] == '.') { // dotdot special case return ads; @@ -293,9 +293,9 @@ namespace #if defined(_WIN32) const auto ads = std::find(filename, last, ':'); // strip alternate data streams in intra-filename decomposition const auto extension = find_extension(filename, ads); -#else // ^^^ _WIN32 / !_WIN32 vvv +#else // ^^^ _WIN32 / !_WIN32 vvv const auto extension = find_extension(filename, last); -#endif // _WIN32 +#endif // _WIN32 return StringView(filename, static_cast(extension - filename)); } @@ -2880,7 +2880,7 @@ namespace vcpkg StatsTimer t(g_us_filesystem_stats); #if defined(_WIN32) return stdfs::create_directories(to_stdfs_path(new_directory), ec); -#else // ^^^ _WIN32 // !_WIN32 vvv +#else // ^^^ _WIN32 // !_WIN32 vvv ec.clear(); if (new_directory.empty()) {