diff --git a/azure-pipelines/e2e-projects/overlays-dot/vcpkg-configuration.json b/azure-pipelines/e2e-projects/overlays-dot/vcpkg-configuration.json new file mode 100644 index 0000000000..8bf3475a84 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-dot/vcpkg-configuration.json @@ -0,0 +1,3 @@ +{ + "overlay-ports": [ "." ] +} diff --git a/azure-pipelines/e2e-projects/overlays-dot/vcpkg.json b/azure-pipelines/e2e-projects/overlays-dot/vcpkg.json new file mode 100644 index 0000000000..bf720c8018 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-dot/vcpkg.json @@ -0,0 +1,5 @@ +{ + "dependencies": [ + "a" + ] +} diff --git a/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/a/portfile.cmake b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/a/portfile.cmake new file mode 100644 index 0000000000..065116c276 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/a/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/a/vcpkg.json b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/a/vcpkg.json new file mode 100644 index 0000000000..dcd0298dbe --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/a/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "a", + "version": "0" +} diff --git a/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/malformed/a/portfile.cmake b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/malformed/a/portfile.cmake new file mode 100644 index 0000000000..c48b693754 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/malformed/a/portfile.cmake @@ -0,0 +1,2 @@ +# This port is intentionally malformed and should not be loaded, because ../../a should be loaded instead +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/malformed/a/vcpkg.json b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/malformed/a/vcpkg.json new file mode 100644 index 0000000000..4383ff0404 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/config-overlays/malformed/a/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "a", + "version": "0", + "intentionally-malformed" +} diff --git a/azure-pipelines/e2e-projects/overlays-malformed-shadowing/vcpkg-configuration.json b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/vcpkg-configuration.json new file mode 100644 index 0000000000..cbf8b52cfe --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/vcpkg-configuration.json @@ -0,0 +1,4 @@ +{ + "$comment": "'config-overlays/a' in . should get loaded first, so we should never consider 'config-overlays/malformed/a'", + "overlay-ports": [ "config-overlays", "config-overlays/malformed" ] +} diff --git a/azure-pipelines/e2e-projects/overlays-malformed-shadowing/vcpkg.json b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/vcpkg.json new file mode 100644 index 0000000000..bf720c8018 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-malformed-shadowing/vcpkg.json @@ -0,0 +1,5 @@ +{ + "dependencies": [ + "a" + ] +} diff --git a/azure-pipelines/e2e-projects/overlays-not-quite-dot/hello/ensure-directory.txt b/azure-pipelines/e2e-projects/overlays-not-quite-dot/hello/ensure-directory.txt new file mode 100644 index 0000000000..00ff321468 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-not-quite-dot/hello/ensure-directory.txt @@ -0,0 +1 @@ +This file exists only to ensure that git creates the directory containing it, so that ./hello/.. is a valid path. \ No newline at end of file diff --git a/azure-pipelines/e2e-projects/overlays-not-quite-dot/vcpkg-configuration.json b/azure-pipelines/e2e-projects/overlays-not-quite-dot/vcpkg-configuration.json new file mode 100644 index 0000000000..dfbb7f3ebc --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-not-quite-dot/vcpkg-configuration.json @@ -0,0 +1,3 @@ +{ + "overlay-ports": [ "./hello/.." ] +} diff --git a/azure-pipelines/e2e-projects/overlays-not-quite-dot/vcpkg.json b/azure-pipelines/e2e-projects/overlays-not-quite-dot/vcpkg.json new file mode 100644 index 0000000000..bf720c8018 --- /dev/null +++ b/azure-pipelines/e2e-projects/overlays-not-quite-dot/vcpkg.json @@ -0,0 +1,5 @@ +{ + "dependencies": [ + "a" + ] +} diff --git a/azure-pipelines/end-to-end-tests-dir/commands.extract.ps1 b/azure-pipelines/end-to-end-tests-dir/commands.extract.ps1 index 366ce00147..f9776b2d5b 100644 --- a/azure-pipelines/end-to-end-tests-dir/commands.extract.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/commands.extract.ps1 @@ -12,8 +12,8 @@ if (-Not (Test-Path $extractedFilePath)) { if (-Not $IsWindows) { $unixMode = (Get-Item $extractedFilePath).UnixMode - if ($unixMode -ne "-rwxr-xr-x") { - throw "File does not have +x permission. UnixMode: $unixMode" + if (-not ($unixMode -match 'x.*x.*x')) { + throw "File does not have +x permission. $extractedFilePath UnixMode: $unixMode" } } diff --git a/azure-pipelines/end-to-end-tests-dir/overlays.ps1 b/azure-pipelines/end-to-end-tests-dir/overlays.ps1 index bf58bbf0cf..9f32317f07 100644 --- a/azure-pipelines/end-to-end-tests-dir/overlays.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/overlays.ps1 @@ -2,9 +2,9 @@ # Tests a simple project with overlay ports and triplets configured on a vcpkg-configuration.json file Copy-Item -Recurse -LiteralPath @( - "$PSScriptRoot/../e2e-projects/overlays-project-with-config", + "$PSScriptRoot/../e2e-projects/overlays-malformed-shadowing", "$PSScriptRoot/../e2e-projects/overlays-project-config-embedded", - "$PSScriptRoot/../e2e-projects/overlays-bad-paths" + "$PSScriptRoot/../e2e-projects/overlays-project-with-config" ) $TestingRoot $manifestRoot = "$TestingRoot/overlays-project-with-config" @@ -27,13 +27,32 @@ Run-Vcpkg install --x-manifest-root=$manifestRoot ` --triplet fancy-config-embedded-triplet Throw-IfFailed +# ... and with command line overlay-ports being 'dot' +pushd "$manifestRoot/cli-overlays" +try { + Run-Vcpkg install --x-manifest-root=$manifestRoot ` + --overlay-ports=. ` + --overlay-triplets=$manifestRoot/my-triplets ` + --x-install-root=$installRoot ` + --triplet fancy-config-embedded-triplet + Throw-IfFailed +} finally { + popd +} + # Config with bad paths -$manifestRoot = "$TestingRoot/overlays-bad-paths" +$manifestRoot = "$PSScriptRoot/../e2e-projects/overlays-bad-paths" $env:VCPKG_OVERLAY_PORTS = "$manifestRoot/env_overlays" Run-Vcpkg install --x-manifest-root=$manifestRoot ` --overlay-triplets=$manifestRoot/my-triplets ` --x-install-root=$installRoot Throw-IfNotFailed +Remove-Item env:VCPKG_OVERLAY_PORTS + +# Test that once an overlay port is loaded for a name, subsequent ports are not considered +$manifestRoot = "$TestingRoot/overlays-malformed-shadowing" +Run-Vcpkg install --x-manifest-root=$manifestRoot +Throw-IfFailed # Test overlay_triplet paths remain relative to the manifest root after x-update-baseline $manifestRoot = "$TestingRoot/overlays-project-with-config" @@ -47,11 +66,27 @@ $overlaysAfter = $configurationAfter."overlay-triplets" $notEqual = @(Compare-Object $overlaysBefore $overlaysAfter -SyncWindow 0).Length -ne 0 if ($notEqual) { - Throw "Overlay triplets paths changed after x-update-baseline" + Throw "Overlay triplets paths changed after x-update-baseline" } +# Test that trying to declare overlay-ports as '.' fails +$manifestRoot = "$PSScriptRoot/../e2e-projects/overlays-dot" +$output = Run-VcpkgAndCaptureStdErr install --x-manifest-root=$manifestRoot --x-install-root=$installRoot +Throw-IfNotFailed +Throw-IfNonContains -Actual $output -Expected @" +error: The manifest directory cannot be the same as a directory configured in overlay-ports, so "overlay-ports" values cannot be ".". +"@ + +# Test that trying to declare overlay-ports as the same directory in a roundabout way fails +$manifestRoot = "$PSScriptRoot/../e2e-projects/overlays-not-quite-dot" +$canonicalManifestRoot = (Get-Item $manifestRoot).FullName +$output = Run-VcpkgAndCaptureStdErr install --x-manifest-root=$manifestRoot --x-install-root=$installRoot +Throw-IfNotFailed +Throw-IfNonContains -Actual $output -Expected @" +The manifest directory ($canonicalManifestRoot) cannot be the same as a directory configured in overlay-ports. +"@ + # Test that removals can happen without the overlay triplets -Remove-Item env:VCPKG_OVERLAY_PORTS Refresh-TestRoot Run-Vcpkg install another-vcpkg-empty-port:fancy-triplet ` --overlay-ports=$PSScriptRoot/../e2e-projects/overlays-project-with-config/cli-overlays ` diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index 3131761a91..3e2ce4e735 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -1120,6 +1120,15 @@ DECLARE_MESSAGE(ErrorInvalidManifestModeOption, (msg::option), "", "The option --{option} is not supported in manifest mode.") +DECLARE_MESSAGE(ErrorManifestMustDifferFromOverlay, + (msg::path), + "", + "The manifest directory ({path}) cannot be the same as a directory configured in overlay-ports.") +DECLARE_MESSAGE(ErrorManifestMustDifferFromOverlayDot, + (), + "", + "The manifest directory cannot be the same as a directory configured in overlay-ports, so " + "\"overlay-ports\" values cannot be \".\".") DECLARE_MESSAGE( ErrorMissingVcpkgRoot, (), @@ -2090,7 +2099,8 @@ DECLARE_MESSAGE(MismatchedManifestAfterReserialize, DECLARE_MESSAGE(MismatchedNames, (msg::package_name, msg::actual), "{actual} is the port name found", - "names did not match: '{package_name}' != '{actual}'") + "the port name declared in the metadata file did not match the directory. Expected the port to be " + "named {package_name}, but the file declares {actual}.") DECLARE_MESSAGE(MismatchedSpec, (msg::path, msg::expected, msg::actual), "{expected} and {actual} are package specs like 'zlib:x64-windows'", @@ -2224,8 +2234,11 @@ DECLARE_MESSAGE(OptionRequiresOption, DECLARE_MESSAGE(Options, (), "Printed just before a list of options for a command", "Options") DECLARE_MESSAGE(OriginalBinParagraphHeader, (), "", "\nOriginal Binary Paragraph") DECLARE_MESSAGE(OtherCommandsHeader, (), "", "Other") -DECLARE_MESSAGE(OverlayPatchDir, (msg::path), "", "Overlay path \"{path}\" must exist and must be a directory.") -DECLARE_MESSAGE(OverlayPortsDirectoriesHelp, (msg::env_var), "", "Directories of overlay ports (also: {env_var})") +DECLARE_MESSAGE(OverlayPatchDir, (msg::path), "", "Overlay path \"{path}\" must be an existing directory.") +DECLARE_MESSAGE(OverlayPortsHelp, + (msg::env_var), + "", + "Overlay-port directories, or directories containing overlay-port directories (also: {env_var})") DECLARE_MESSAGE(OverlayTripletDirectoriesHelp, (msg::env_var), "", "Directories of overlay triplets (also: {env_var})") DECLARE_MESSAGE(OverlayTriplets, (msg::path), "", "Overlay Triplets from \"{path}\":") DECLARE_MESSAGE(OverwritingFile, (msg::path), "", "File {path} was already present and will be overwritten") diff --git a/include/vcpkg/commands.find.h b/include/vcpkg/commands.find.h index f363f60ab0..ed6baac1c1 100644 --- a/include/vcpkg/commands.find.h +++ b/include/vcpkg/commands.find.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -15,7 +16,7 @@ namespace vcpkg bool full_description, bool enable_json, Optional filter, - View overlay_ports); + const OverlayPortPaths& overlay_ports); extern const CommandMetadata CommandFindMetadata; void command_find_and_exit(const VcpkgCmdArguments& args, const VcpkgPaths& paths); } diff --git a/include/vcpkg/fwd/portfileprovider.h b/include/vcpkg/fwd/portfileprovider.h index 22869bb90e..2c93a1b66e 100644 --- a/include/vcpkg/fwd/portfileprovider.h +++ b/include/vcpkg/fwd/portfileprovider.h @@ -2,9 +2,18 @@ namespace vcpkg { + struct OverlayPortPaths; + struct OverlayPortIndexEntry; struct PortFileProvider; struct PathsPortFileProvider; struct IVersionedPortfileProvider; struct IBaselineProvider; struct IOverlayProvider; + + enum class OverlayPortKind + { + Unknown, // We have not tested yet + Port, // The overlay directory is itself a port + Directory, // The overlay directory itself contains port directories + }; } diff --git a/include/vcpkg/paragraphs.h b/include/vcpkg/paragraphs.h index bdb7e9b5de..4770f65b8a 100644 --- a/include/vcpkg/paragraphs.h +++ b/include/vcpkg/paragraphs.h @@ -27,8 +27,6 @@ namespace vcpkg::Paragraphs void append_paragraph_field(StringView name, StringView field, std::string& out_str); - bool is_port_directory(const ReadOnlyFilesystem& fs, const Path& maybe_directory); - struct PortLoadResult { ExpectedL maybe_scfl; @@ -63,7 +61,4 @@ namespace vcpkg::Paragraphs LoadResults try_load_all_registry_ports(const RegistrySet& registries); std::vector load_all_registry_ports(const RegistrySet& registries); - - LoadResults try_load_overlay_ports(const ReadOnlyFilesystem& fs, const Path& dir); - std::vector load_overlay_ports(const ReadOnlyFilesystem& fs, const Path& dir); } diff --git a/include/vcpkg/portfileprovider.h b/include/vcpkg/portfileprovider.h index 25a87ebff7..312e26d66f 100644 --- a/include/vcpkg/portfileprovider.h +++ b/include/vcpkg/portfileprovider.h @@ -5,12 +5,66 @@ #include #include +#include #include -#include +#include + +#include +#include +#include +#include namespace vcpkg { + struct OverlayPortPaths + { + Optional builtin_overlay_port_dir; + std::vector overlay_ports; + + bool empty() const noexcept; + }; + + struct OverlayPortIndexEntry + { + OverlayPortIndexEntry(OverlayPortKind kind, const Path& directory); + OverlayPortIndexEntry(const OverlayPortIndexEntry&) = delete; + OverlayPortIndexEntry(OverlayPortIndexEntry&&); + + const ExpectedL* try_load_port(const ReadOnlyFilesystem& fs, + StringView port_name); + + ExpectedL try_load_all_ports(const ReadOnlyFilesystem& fs, + std::map& out); + + void check_directory(const ReadOnlyFilesystem& fs) const; + + private: + OverlayPortKind m_kind; + Path m_directory; + + using MapT = std::map, std::less<>>; + // If kind == OverlayPortKind::Unknown, empty + // Otherwise, if kind == OverlayPortKind::Port, + // upon load success, contains exactly one entry with the loaded name of the port + // upon load failure, contains exactly one entry with a key of empty string, value being the load error + // Otherwise, if kind == OverlayPortKind::Directory, contains an entry for each loaded overlay-port in the + // directory + MapT m_loaded_ports; + + OverlayPortKind determine_kind(const ReadOnlyFilesystem& fs); + const ExpectedL* try_load_port_cached_port(StringView port_name); + + MapT::iterator try_load_port_subdirectory_uncached(MapT::iterator hint, + const ReadOnlyFilesystem& fs, + StringView port_name); + + ExpectedL load_all_port_subdirectories(const ReadOnlyFilesystem& fs); + + const ExpectedL* try_load_port_subdirectory_with_cache( + const ReadOnlyFilesystem& fs, StringView port_name); + }; + struct PortFileProvider { virtual ~PortFileProvider() = default; @@ -76,9 +130,10 @@ namespace vcpkg std::unique_ptr make_baseline_provider(const RegistrySet& registry_set); std::unique_ptr make_versioned_portfile_provider(const RegistrySet& registry_set); - std::unique_ptr make_overlay_provider(const ReadOnlyFilesystem& fs, View overlay_ports); + std::unique_ptr make_overlay_provider(const ReadOnlyFilesystem& fs, + const OverlayPortPaths& overlay_ports); std::unique_ptr make_manifest_provider(const ReadOnlyFilesystem& fs, - View overlay_ports, + const OverlayPortPaths& overlay_ports, const Path& manifest_path, std::unique_ptr&& manifest_scf); } diff --git a/include/vcpkg/vcpkgpaths.h b/include/vcpkg/vcpkgpaths.h index 0639a5b8cc..d1309723c7 100644 --- a/include/vcpkg/vcpkgpaths.h +++ b/include/vcpkg/vcpkgpaths.h @@ -21,6 +21,8 @@ #include #include +#include + #include #include #include @@ -101,7 +103,7 @@ namespace vcpkg std::vector overlay_triplets; public: - std::vector overlay_ports; + OverlayPortPaths overlay_ports; std::string get_toolver_diagnostics() const; diff --git a/locales/messages.json b/locales/messages.json index d33ef52815..a7e03c113f 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -650,6 +650,9 @@ "_ErrorInvalidExtractOption.comment": "The keyword 'AUTO' should not be localized An example of {option} is editable.", "ErrorInvalidManifestModeOption": "The option --{option} is not supported in manifest mode.", "_ErrorInvalidManifestModeOption.comment": "An example of {option} is editable.", + "ErrorManifestMustDifferFromOverlay": "The manifest directory ({path}) cannot be the same as a directory configured in overlay-ports.", + "_ErrorManifestMustDifferFromOverlay.comment": "An example of {path} is /foo/bar.", + "ErrorManifestMustDifferFromOverlayDot": "The manifest directory cannot be the same as a directory configured in overlay-ports, so \"overlay-ports\" values cannot be \".\".", "ErrorMissingVcpkgRoot": "Could not detect vcpkg-root. If you are trying to use a copy of vcpkg that you've built, you must define the VCPKG_ROOT environment variable to point to a cloned copy of https://github.com/Microsoft/vcpkg.", "ErrorNoVSInstance": "in triplet {triplet}: Unable to find a valid Visual Studio instance", "_ErrorNoVSInstance.comment": "An example of {triplet} is x64-windows.", @@ -1143,7 +1146,7 @@ "MismatchedFiles": "file to store does not match hash", "MismatchedManifestAfterReserialize": "The serialized manifest was different from the original manifest. Please open an issue at https://github.com/microsoft/vcpkg, with the following output:", "_MismatchedManifestAfterReserialize.comment": "The original file output and generated output are printed after this line, in English as it's intended to be used in the issue submission and read by devs. This message indicates an internal error in vcpkg.", - "MismatchedNames": "names did not match: '{package_name}' != '{actual}'", + "MismatchedNames": "the port name declared in the metadata file did not match the directory. Expected the port to be named {package_name}, but the file declares {actual}.", "_MismatchedNames.comment": "{actual} is the port name found An example of {package_name} is zlib.", "MismatchedSpec": "Mismatched spec in port {path}: expected {expected}, actual {actual}", "_MismatchedSpec.comment": "{expected} and {actual} are package specs like 'zlib:x64-windows' An example of {path} is /foo/bar.", @@ -1228,10 +1231,10 @@ "_Options.comment": "Printed just before a list of options for a command", "OriginalBinParagraphHeader": "\nOriginal Binary Paragraph", "OtherCommandsHeader": "Other", - "OverlayPatchDir": "Overlay path \"{path}\" must exist and must be a directory.", + "OverlayPatchDir": "Overlay path \"{path}\" must be an existing directory.", "_OverlayPatchDir.comment": "An example of {path} is /foo/bar.", - "OverlayPortsDirectoriesHelp": "Directories of overlay ports (also: {env_var})", - "_OverlayPortsDirectoriesHelp.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", + "OverlayPortsHelp": "Overlay-port directories, or directories containing overlay-port directories (also: {env_var})", + "_OverlayPortsHelp.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", "OverlayTripletDirectoriesHelp": "Directories of overlay triplets (also: {env_var})", "_OverlayTripletDirectoriesHelp.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", "OverlayTriplets": "Overlay Triplets from \"{path}\":", diff --git a/src/vcpkg/commands.build-external.cpp b/src/vcpkg/commands.build-external.cpp index 28720fa55b..3bf6987067 100644 --- a/src/vcpkg/commands.build-external.cpp +++ b/src/vcpkg/commands.build-external.cpp @@ -43,7 +43,7 @@ namespace vcpkg .value_or_exit(VCPKG_LINE_INFO); auto overlays = paths.overlay_ports; - overlays.insert(overlays.begin(), options.command_arguments[1]); + overlays.overlay_ports.insert(overlays.overlay_ports.begin(), options.command_arguments[1]); auto& fs = paths.get_filesystem(); auto registry_set = paths.make_registry_set(); diff --git a/src/vcpkg/commands.find.cpp b/src/vcpkg/commands.find.cpp index d40a307193..a996f83309 100644 --- a/src/vcpkg/commands.find.cpp +++ b/src/vcpkg/commands.find.cpp @@ -133,7 +133,7 @@ namespace vcpkg bool full_description, bool enable_json, Optional filter, - View overlay_ports) + const OverlayPortPaths& overlay_ports) { Checks::check_exit(VCPKG_LINE_INFO, msg::default_output_stream == OutputStream::StdErr); auto& fs = paths.get_filesystem(); diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 5f750f3658..43f9e5787b 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -1236,16 +1236,14 @@ namespace vcpkg auto verprovider = make_versioned_portfile_provider(*registry_set); auto baseprovider = make_baseline_provider(*registry_set); - std::vector extended_overlay_ports; - extended_overlay_ports.reserve(paths.overlay_ports.size() + add_builtin_ports_directory_as_overlay); - extended_overlay_ports = paths.overlay_ports; + auto extended_overlay_port_directories = paths.overlay_ports; if (add_builtin_ports_directory_as_overlay) { - extended_overlay_ports.emplace_back(paths.builtin_ports_directory()); + extended_overlay_port_directories.builtin_overlay_port_dir.emplace(paths.builtin_ports_directory()); } auto oprovider = - make_manifest_provider(fs, extended_overlay_ports, manifest->path, std::move(manifest_scf)); + make_manifest_provider(fs, extended_overlay_port_directories, manifest->path, std::move(manifest_scf)); auto install_plan = create_versioned_install_plan(*verprovider, *baseprovider, *oprovider, diff --git a/src/vcpkg/commands.portsdiff.cpp b/src/vcpkg/commands.portsdiff.cpp index d8fdd046fc..5eddf09d10 100644 --- a/src/vcpkg/commands.portsdiff.cpp +++ b/src/vcpkg/commands.portsdiff.cpp @@ -54,16 +54,15 @@ namespace paths.git_cmd_builder(dot_git_dir, temp_checkout_path).string_arg("reset"), settings), Tools::GIT) .value_or_exit(VCPKG_LINE_INFO); - const auto ports_at_commit = Paragraphs::load_overlay_ports(fs, temp_checkout_path / ports_dir_name); - fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO); - - auto results = Util::fmap(ports_at_commit, [](const SourceControlFileAndLocation& scfl) { - return scfl.source_control_file->to_version_spec(); - }); - Util::sort(results, - [](const VersionSpec& lhs, const VersionSpec& rhs) { return lhs.port_name < rhs.port_name; }); - return results; + OverlayPortIndexEntry ports_at_commit_index(OverlayPortKind::Directory, temp_checkout_path / ports_dir_name); + std::map ports_at_commit; + ports_at_commit_index.try_load_all_ports(fs, ports_at_commit).value_or_exit(VCPKG_LINE_INFO); + fs.remove_all(temp_checkout_path, VCPKG_LINE_INFO); + return Util::fmap(ports_at_commit, + [](const std::pair& cache_entry) { + return cache_entry.second->source_control_file->to_version_spec(); + }); } void check_commit_exists(const VcpkgPaths& paths, StringView git_commit_id) diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index e203799b16..ff6218ccab 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -362,12 +362,6 @@ namespace vcpkg::Paragraphs out_str.append(name.data(), name.size()).append(": ").append(field.data(), field.size()).push_back('\n'); } - bool is_port_directory(const ReadOnlyFilesystem& fs, const Path& maybe_directory) - { - return fs.exists(maybe_directory / "CONTROL", IgnoreErrors{}) || - fs.exists(maybe_directory / "vcpkg.json", IgnoreErrors{}); - } - ExpectedL> try_load_project_manifest_text(StringView text, StringView control_path, MessageSink& warning_sink) @@ -588,41 +582,5 @@ namespace vcpkg::Paragraphs return std::move(results.paragraphs); } - LoadResults try_load_overlay_ports(const ReadOnlyFilesystem& fs, const Path& directory) - { - LoadResults ret; - - auto port_dirs = fs.get_directories_non_recursive(directory, VCPKG_LINE_INFO); - Util::sort(port_dirs); - - Util::erase_remove_if(port_dirs, - [&](auto&& port_dir_entry) { return port_dir_entry.filename() == FileDotDsStore; }); - - for (auto&& path : port_dirs) - { - auto port_name = path.filename(); - auto maybe_spgh = try_load_port_required(fs, port_name, PortLocation{path}).maybe_scfl; - if (const auto spgh = maybe_spgh.get()) - { - ret.paragraphs.push_back(std::move(*spgh)); - } - else - { - ret.errors.emplace_back(std::piecewise_construct, - std::forward_as_tuple(port_name.data(), port_name.size()), - std::forward_as_tuple(std::move(maybe_spgh).error())); - } - } - - return ret; - } - - std::vector load_overlay_ports(const ReadOnlyFilesystem& fs, const Path& directory) - { - auto results = try_load_overlay_ports(fs, directory); - load_results_print_error(results); - return std::move(results.paragraphs); - } - uint64_t get_load_ports_stats() { return g_load_ports_stats.load(); } } diff --git a/src/vcpkg/portfileprovider.cpp b/src/vcpkg/portfileprovider.cpp index 8d9c904c20..04c1c9f428 100644 --- a/src/vcpkg/portfileprovider.cpp +++ b/src/vcpkg/portfileprovider.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -10,10 +11,309 @@ #include #include +#include +#include + using namespace vcpkg; namespace vcpkg { + OverlayPortIndexEntry::OverlayPortIndexEntry(OverlayPortKind kind, const Path& directory) + : m_kind(kind), m_directory(directory), m_loaded_ports() + { + if (m_kind == OverlayPortKind::Port) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + } + + OverlayPortIndexEntry::OverlayPortIndexEntry(OverlayPortIndexEntry&&) = default; + + OverlayPortKind OverlayPortIndexEntry::determine_kind(const ReadOnlyFilesystem& fs) + { + if (m_kind == OverlayPortKind::Unknown) + { + if (!m_loaded_ports.empty()) + { + Checks::unreachable(VCPKG_LINE_INFO, "OverlayPortKind::Unknown empty cache constraint violated"); + } + + auto maybe_scfl = Paragraphs::try_load_port(fs, PortLocation{m_directory}).maybe_scfl; + if (auto scfl = maybe_scfl.get()) + { + if (scfl->source_control_file) + { + // succeeded in loading it, so this must be a port + m_kind = OverlayPortKind::Port; + auto name_copy = scfl->to_name(); // copy name before moving maybe_scfl + m_loaded_ports.emplace(name_copy, std::move(maybe_scfl)); + } + else + { + // the directory didn't look like a port at all, consider it an overlay-port-dir + m_kind = OverlayPortKind::Directory; + } + } + else + { + // it looked like a port but we failed to load it for some reason + m_kind = OverlayPortKind::Port; + m_loaded_ports.emplace(std::string(), std::move(maybe_scfl)); + } + } + + return m_kind; + } + + const ExpectedL* OverlayPortIndexEntry::try_load_port_cached_port( + StringView port_name) + { + if (m_kind != OverlayPortKind::Port) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + const auto& this_overlay = *m_loaded_ports.begin(); + if (auto scfl = this_overlay.second.get()) + { + if (scfl->to_name() != port_name) + { + return nullptr; // this overlay-port is OK, but this isn't the right one + } + } + + return &this_overlay.second; + } + + OverlayPortIndexEntry::MapT::iterator OverlayPortIndexEntry::try_load_port_subdirectory_uncached( + MapT::iterator hint, const ReadOnlyFilesystem& fs, StringView port_name) + { + if (m_kind != OverlayPortKind::Directory) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + auto port_directory = m_directory / port_name; + auto load_result = Paragraphs::try_load_port(fs, PortLocation{port_directory}); + auto& maybe_scfl = load_result.maybe_scfl; + if (auto scfl = maybe_scfl.get()) + { + if (auto scf = scfl->source_control_file.get()) + { + const auto& actual_name = scf->to_name(); + if (actual_name != port_name) + { + maybe_scfl = + LocalizedString::from_raw(scfl->control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgMismatchedNames, msg::package_name = port_name, msg::actual = actual_name); + } + } + else + { + return m_loaded_ports.end(); + } + } + + return m_loaded_ports.emplace_hint(hint, port_name.to_string(), std::move(maybe_scfl)); + } + + const ExpectedL* OverlayPortIndexEntry::try_load_port_subdirectory_with_cache( + const ReadOnlyFilesystem& fs, StringView port_name) + { + if (m_kind != OverlayPortKind::Directory) + { + Checks::unreachable(VCPKG_LINE_INFO); + } + + auto already_loaded = m_loaded_ports.lower_bound(port_name); + if (already_loaded == m_loaded_ports.end() || already_loaded->first != port_name) + { + already_loaded = try_load_port_subdirectory_uncached(already_loaded, fs, port_name); + } + + if (already_loaded == m_loaded_ports.end()) + { + return nullptr; + } + + return &already_loaded->second; + } + + const ExpectedL* OverlayPortIndexEntry::try_load_port(const ReadOnlyFilesystem& fs, + StringView port_name) + { + switch (determine_kind(fs)) + { + case OverlayPortKind::Port: return try_load_port_cached_port(port_name); + case OverlayPortKind::Directory: return try_load_port_subdirectory_with_cache(fs, port_name); + case OverlayPortKind::Unknown: + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + ExpectedL OverlayPortIndexEntry::try_load_all_ports( + const ReadOnlyFilesystem& fs, std::map& out) + { + switch (determine_kind(fs)) + { + case OverlayPortKind::Port: + { + auto& maybe_this_port = *m_loaded_ports.begin(); + if (auto this_port = maybe_this_port.second.get()) + { + auto already_in_out = out.lower_bound(maybe_this_port.first); + if (already_in_out == out.end() || already_in_out->first != maybe_this_port.first) + { + out.emplace_hint(already_in_out, maybe_this_port.first, this_port); + } + + return Unit{}; + } + + return maybe_this_port.second.error(); + } + case OverlayPortKind::Directory: + { + auto maybe_subdirectories = fs.try_get_directories_non_recursive(m_directory); + if (auto subdirectories = maybe_subdirectories.get()) + { + std::vector errors; + Util::sort(*subdirectories); + auto first_loaded = m_loaded_ports.begin(); + const auto last_loaded = m_loaded_ports.end(); + auto first_out = out.begin(); + const auto last_out = out.end(); + for (const auto& full_subdirectory : *subdirectories) + { + auto subdirectory = full_subdirectory.filename(); + while (first_out != last_out && first_out->first < subdirectory) + { + ++first_out; + } + + if (first_out != last_out && first_out->first == subdirectory) + { + // this subdirectory is already in the output; we shouldn't replace or attempt to load it + ++first_out; + continue; + } + + while (first_loaded != last_loaded && first_loaded->first < subdirectory) + { + ++first_loaded; + } + + if (first_loaded == last_loaded || first_loaded->first != subdirectory) + { + // the subdirectory isn't cached, load it into the cache + first_loaded = try_load_port_subdirectory_uncached(first_loaded, fs, subdirectory); + } // else: the subdirectory is already loaded + + if (auto this_port = first_loaded->second.get()) + { + first_out = out.emplace_hint(first_out, first_loaded->first, this_port); + ++first_out; + } + else + { + errors.push_back(first_loaded->second.error()); + } + + ++first_loaded; + } + + if (errors.empty()) + { + return Unit{}; + } + + return LocalizedString::from_raw(Strings::join("\n", errors)); + } + + return maybe_subdirectories.error(); + } + case OverlayPortKind::Unknown: + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + void OverlayPortIndexEntry::check_directory(const ReadOnlyFilesystem& fs) const + { + Debug::println("Using overlay: ", m_directory); + + Checks::msg_check_exit(VCPKG_LINE_INFO, + vcpkg::is_directory(fs.status(m_directory, VCPKG_LINE_INFO)), + msgOverlayPatchDir, + msg::path = m_directory); + } + + struct OverlayPortIndex + { + OverlayPortIndex() = delete; + OverlayPortIndex(const OverlayPortIndex&) = delete; + OverlayPortIndex(OverlayPortIndex&&) = default; + + OverlayPortIndex(const OverlayPortPaths& paths) + { + if (auto builtin_overlay_port_dir = paths.builtin_overlay_port_dir.get()) + { + m_entries.emplace_back(OverlayPortKind::Directory, *builtin_overlay_port_dir); + } + + for (auto&& overlay_port : paths.overlay_ports) + { + m_entries.emplace_back(OverlayPortKind::Unknown, overlay_port); + } + } + + const ExpectedL* try_load_port(const ReadOnlyFilesystem& fs, StringView port_name) + { + for (auto&& entry : m_entries) + { + auto result = entry.try_load_port(fs, port_name); + if (result) + { + return result; + } + } + + return nullptr; + } + + ExpectedL try_load_all_ports(const ReadOnlyFilesystem& fs, + std::map& out) + { + for (auto&& entry : m_entries) + { + auto result = entry.try_load_all_ports(fs, out); + if (!result) + { + return result; + } + } + + return Unit{}; + } + + void check_directories(const ReadOnlyFilesystem& fs) + { + for (auto&& overlay : m_entries) + { + overlay.check_directory(fs); + } + } + + private: + std::vector m_entries; + }; + + bool OverlayPortPaths::empty() const noexcept + { + return !builtin_overlay_port_dir.has_value() && overlay_ports.empty(); + } + MapPortFileProvider::MapPortFileProvider(const std::unordered_map& map) : ports(map) { @@ -211,158 +511,40 @@ namespace vcpkg struct OverlayProviderImpl : IFullOverlayProvider { - OverlayProviderImpl(const ReadOnlyFilesystem& fs, View overlay_ports) - : m_fs(fs), m_overlay_ports(overlay_ports.begin(), overlay_ports.end()) + OverlayProviderImpl(const ReadOnlyFilesystem& fs, const OverlayPortPaths& overlay_port_paths) + : m_fs(fs), m_overlay_index(overlay_port_paths) { - for (auto&& overlay : m_overlay_ports) - { - Debug::println("Using overlay: ", overlay); - - Checks::msg_check_exit(VCPKG_LINE_INFO, - vcpkg::is_directory(m_fs.status(overlay, VCPKG_LINE_INFO)), - msgOverlayPatchDir, - msg::path = overlay); - } + m_overlay_index.check_directories(m_fs); } OverlayProviderImpl(const OverlayProviderImpl&) = delete; OverlayProviderImpl& operator=(const OverlayProviderImpl&) = delete; - - Optional load_port(StringView port_name) const - { - auto s_port_name = port_name.to_string(); - - for (auto&& ports_dir : m_overlay_ports) - { - // Try loading individual port - if (Paragraphs::is_port_directory(m_fs, ports_dir)) - { - auto maybe_scfl = - Paragraphs::try_load_port_required(m_fs, port_name, PortLocation{ports_dir}).maybe_scfl; - if (auto scfl = maybe_scfl.get()) - { - if (scfl->to_name() == port_name) - { - return std::move(*scfl); - } - } - else - { - print_error_message(maybe_scfl.error()); - msg::println(); - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO); - } - - continue; - } - - auto ports_spec = ports_dir / port_name; - if (Paragraphs::is_port_directory(m_fs, ports_spec)) - { - auto found_scfl = - Paragraphs::try_load_port_required(m_fs, port_name, PortLocation{ports_spec}).maybe_scfl; - if (auto scfl = found_scfl.get()) - { - auto& scfl_name = scfl->to_name(); - if (scfl_name == port_name) - { - return std::move(*scfl); - } - - Checks::msg_exit_maybe_upgrade(VCPKG_LINE_INFO, - LocalizedString::from_raw(ports_spec) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgMismatchedNames, - msg::package_name = port_name, - msg::actual = scfl_name)); - } - else - { - print_error_message(found_scfl.error()); - msg::println(); - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO); - } - } - } - return nullopt; - } - virtual Optional get_control_file(StringView port_name) const override { - auto it = m_overlay_cache.find(port_name); - if (it == m_overlay_cache.end()) + auto loaded = m_overlay_index.try_load_port(m_fs, port_name); + if (loaded) { - it = m_overlay_cache.emplace(port_name.to_string(), load_port(port_name)).first; + return loaded->value_or_exit(VCPKG_LINE_INFO); } - return it->second; + + return nullopt; } virtual void load_all_control_files( std::map& out) const override { - auto first = std::make_reverse_iterator(m_overlay_ports.end()); - const auto last = std::make_reverse_iterator(m_overlay_ports.begin()); - for (; first != last; ++first) - { - auto&& ports_dir = *first; - // Try loading individual port - if (Paragraphs::is_port_directory(m_fs, ports_dir)) - { - auto maybe_scfl = - Paragraphs::try_load_port_required(m_fs, ports_dir.filename(), PortLocation{ports_dir}) - .maybe_scfl; - if (auto scfl = maybe_scfl.get()) - { - // copy name before moving *scfl - auto name = scfl->to_name(); - auto it = m_overlay_cache.emplace(std::move(name), std::move(*scfl)).first; - Checks::check_exit(VCPKG_LINE_INFO, it->second.get()); - out.emplace(it->first, it->second.get()); - } - else - { - print_error_message(maybe_scfl.error()); - msg::println(); - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO); - } - - continue; - } - - // Try loading all ports inside ports_dir - auto results = Paragraphs::try_load_overlay_ports(m_fs, ports_dir); - if (!results.errors.empty()) - { - print_error_message(LocalizedString::from_raw(Strings::join( - "\n", - results.errors, - [](const std::pair& err) -> const LocalizedString& { - return err.second; - }))); - Checks::exit_maybe_upgrade(VCPKG_LINE_INFO); - } - - for (auto&& scfl : results.paragraphs) - { - auto name = scfl.to_name(); - auto it = m_overlay_cache.emplace(std::move(name), std::move(scfl)).first; - Checks::check_exit(VCPKG_LINE_INFO, it->second.get()); - out.emplace(it->first, it->second.get()); - } - } + m_overlay_index.try_load_all_ports(m_fs, out).value_or_exit(VCPKG_LINE_INFO); } private: const ReadOnlyFilesystem& m_fs; - const std::vector m_overlay_ports; - mutable std::map, std::less<>> m_overlay_cache; + mutable OverlayPortIndex m_overlay_index; }; struct ManifestProviderImpl : IFullOverlayProvider { ManifestProviderImpl(const ReadOnlyFilesystem& fs, - View overlay_ports, + const OverlayPortPaths& overlay_ports, const Path& manifest_path, std::unique_ptr&& manifest_scf) : m_overlay_ports{fs, overlay_ports} @@ -404,13 +586,14 @@ namespace vcpkg return std::make_unique(registry_set); } - std::unique_ptr make_overlay_provider(const ReadOnlyFilesystem& fs, View overlay_ports) + std::unique_ptr make_overlay_provider(const ReadOnlyFilesystem& fs, + const OverlayPortPaths& overlay_ports) { return std::make_unique(fs, overlay_ports); } std::unique_ptr make_manifest_provider(const ReadOnlyFilesystem& fs, - View overlay_ports, + const OverlayPortPaths& overlay_ports, const Path& manifest_path, std::unique_ptr&& manifest_scf) { diff --git a/src/vcpkg/vcpkgcmdarguments.cpp b/src/vcpkg/vcpkgcmdarguments.cpp index 446466079d..7c9c033be2 100644 --- a/src/vcpkg/vcpkgcmdarguments.cpp +++ b/src/vcpkg/vcpkgcmdarguments.cpp @@ -379,7 +379,7 @@ namespace vcpkg SwitchOverlayPorts, StabilityTag::Standard, args.cli_overlay_ports, - msg::format(msgOverlayPortsDirectoriesHelp, + msg::format(msgOverlayPortsHelp, msg::env_var = format_environment_variable(EnvironmentVariableVcpkgOverlayPorts))); args.parser.parse_multi_option( SwitchOverlayTriplets, diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index fe7748d6ca..8261e1d40b 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -99,29 +99,46 @@ namespace } static void append_overlays(std::vector& result, + const ReadOnlyFilesystem& fs, const std::vector& overlay_entries, - const Path& relative_root) + const Path& relative_root, + const Path& config_directory, + bool forbid_dot) { for (auto&& entry : overlay_entries) { - result.push_back(relative_root / entry); + if (forbid_dot && entry == ".") + { + Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgErrorManifestMustDifferFromOverlayDot); + } + + auto full_entry = relative_root / entry; + if (forbid_dot && (fs.almost_canonical(full_entry, VCPKG_LINE_INFO) / "") == (config_directory / "")) + { + Checks::msg_exit_with_error( + VCPKG_LINE_INFO, msgErrorManifestMustDifferFromOverlay, msg::path = config_directory); + } + + result.push_back(std::move(full_entry)); } } // Merges overlay settings from the 3 major sources in the usual priority order, where command line wins first, then // manifest, then environment. The parameter order is specifically chosen to group information that comes from the // manifest together and make parameter order confusion less likely to compile. - static std::vector merge_overlays(const std::vector& cli_overlays, + static std::vector merge_overlays(const ReadOnlyFilesystem& fs, + const std::vector& cli_overlays, const std::vector& env_overlays, const Path& original_cwd, + bool forbid_config_dot, const std::vector& config_overlays, const Path& config_directory) { std::vector result; result.reserve(cli_overlays.size() + config_overlays.size() + env_overlays.size()); - append_overlays(result, cli_overlays, original_cwd); - append_overlays(result, config_overlays, config_directory); - append_overlays(result, env_overlays, original_cwd); + append_overlays(result, fs, cli_overlays, original_cwd, config_directory, false); + append_overlays(result, fs, config_overlays, config_directory, config_directory, forbid_config_dot); + append_overlays(result, fs, env_overlays, original_cwd, config_directory, false); return result; } @@ -688,14 +705,18 @@ namespace vcpkg std::move(maybe_json_config), m_pimpl->m_config_dir, *this); - overlay_ports = merge_overlays(args.cli_overlay_ports, - args.env_overlay_ports, - original_cwd, - m_pimpl->m_config.config.overlay_ports, - m_pimpl->m_config.directory); - overlay_triplets = merge_overlays(args.cli_overlay_triplets, + overlay_ports.overlay_ports = merge_overlays(m_pimpl->m_fs, + args.cli_overlay_ports, + args.env_overlay_ports, + original_cwd, + true, + m_pimpl->m_config.config.overlay_ports, + m_pimpl->m_config.directory); + overlay_triplets = merge_overlays(m_pimpl->m_fs, + args.cli_overlay_triplets, args.env_overlay_triplets, original_cwd, + false, m_pimpl->m_config.config.overlay_triplets, m_pimpl->m_config.directory); for (const auto& triplet : this->overlay_triplets)