Skip to content

Commit

Permalink
bzlmod: Precompute data structures needed for module extension resolu…
Browse files Browse the repository at this point in the history
…tion

(bazelbuild#13316)

At the end of BazelModuleResolutionFunction, we do some precomputation such as grouping all ModuleExtensionUsages by their "extension ID", and calculating a unique name for each extension.

PiperOrigin-RevId: 395948289
  • Loading branch information
Wyverald authored and copybara-github committed Sep 10, 2021
1 parent 83cd456 commit 3aaddca
Show file tree
Hide file tree
Showing 12 changed files with 377 additions and 93 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;

/**
* An abridged version of a {@link Module}, with a reduced set of information available, used for
* module extension resolution.
*/
@AutoValue
public abstract class AbridgedModule {
public abstract String getName();

public abstract Version getVersion();

public abstract ModuleKey getKey();

public final String getCanonicalRepoName() {
return getKey().getCanonicalRepoName();
}

public abstract ImmutableMap<String, ModuleKey> getDeps();

public final RepositoryMapping getRepoMapping() {
return Module.getRepoMappingWithBazelDepsOnly(getKey(), getName(), getDeps());
}

public static AbridgedModule from(Module module) {
return new AutoValue_AbridgedModule(
module.getName(), module.getVersion(), module.getKey(), module.getDeps());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ java_library(
name = "module_extension",
srcs = [
"ModuleExtension.java",
"ModuleExtensionId.java",
"ModuleExtensionUsage.java",
"Tag.java",
"TagClass.java",
Expand Down Expand Up @@ -81,6 +82,7 @@ java_library(
java_library(
name = "resolution",
srcs = [
"AbridgedModule.java",
"ArchiveOverride.java",
"BazelModuleResolutionValue.java",
"GitOverride.java",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,24 @@

package com.google.devtools.build.lib.bazel.bzlmod;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.packages.BuildType.LabelConversionContext;
import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.util.HashMap;

/**
* Runs Bazel module resolution. This function produces the dependency graph containing all Bazel
Expand Down Expand Up @@ -57,7 +65,9 @@ public SkyValue compute(SkyKey skyKey, Environment env)

@VisibleForTesting
static BazelModuleResolutionValue createValue(
ImmutableMap<ModuleKey, Module> depGraph, ImmutableMap<String, ModuleOverride> overrides) {
ImmutableMap<ModuleKey, Module> depGraph, ImmutableMap<String, ModuleOverride> overrides)
throws BazelModuleResolutionFunctionException {
// Build some reverse lookups for later use.
ImmutableMap<String, ModuleKey> canonicalRepoNameLookup =
depGraph.keySet().stream()
.collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key));
Expand All @@ -69,7 +79,58 @@ static BazelModuleResolutionValue createValue(
.filter(key -> !(overrides.get(key.getName()) instanceof MultipleVersionOverride))
.collect(toImmutableMap(ModuleKey::getName, key -> key));

return BazelModuleResolutionValue.create(depGraph, canonicalRepoNameLookup, moduleNameLookup);
// For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can
// group all usages by the label + name (the ModuleExtensionId).
ImmutableTable.Builder<ModuleExtensionId, ModuleKey, ModuleExtensionUsage>
extensionUsagesTableBuilder = ImmutableTable.builder();
for (Module module : depGraph.values()) {
LabelConversionContext labelConversionContext =
new LabelConversionContext(
StarlarkBazelModule.createModuleRootLabel(module.getCanonicalRepoName()),
module.getRepoMappingWithBazelDepsOnly(),
new HashMap<>());
for (ModuleExtensionUsage usage : module.getExtensionUsages()) {
try {
ModuleExtensionId moduleExtensionId =
ModuleExtensionId.create(
labelConversionContext.convert(usage.getExtensionBzlFile()),
usage.getExtensionName());
extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage);
} catch (LabelSyntaxException e) {
throw new BazelModuleResolutionFunctionException(
ExternalDepsException.withCauseAndMessage(
Code.BAD_MODULE,
e,
"invalid label for module extension found at %s",
usage.getLocation()),
Transience.PERSISTENT);
}
}
}
ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesById =
extensionUsagesTableBuilder.build();

// Calculate a unique name for each used extension id.
BiMap<String, ModuleExtensionId> extensionUniqueNames = HashBiMap.create();
for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) {
String bestName =
id.getBzlFileLabel().getRepository().strippedName() + "." + id.getExtensionName();
if (extensionUniqueNames.putIfAbsent(bestName, id) == null) {
continue;
}
int suffix = 2;
while (extensionUniqueNames.putIfAbsent(bestName + suffix, id) != null) {
suffix++;
}
}

return BazelModuleResolutionValue.create(
depGraph,
canonicalRepoNameLookup,
moduleNameLookup,
depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()),
extensionUsagesById,
ImmutableMap.copyOf(extensionUniqueNames.inverse()));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,16 @@
package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.util.Map;

/**
* The result of running Bazel module resolution, containing the Bazel module dependency graph
Expand All @@ -33,9 +38,17 @@ public abstract class BazelModuleResolutionValue implements SkyValue {
public static BazelModuleResolutionValue create(
ImmutableMap<ModuleKey, Module> depGraph,
ImmutableMap<String, ModuleKey> canonicalRepoNameLookup,
ImmutableMap<String, ModuleKey> moduleNameLookup) {
ImmutableMap<String, ModuleKey> moduleNameLookup,
ImmutableList<AbridgedModule> abridgedModules,
ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage> extensionUsagesTable,
ImmutableMap<ModuleExtensionId, String> extensionUniqueNames) {
return new AutoValue_BazelModuleResolutionValue(
depGraph, canonicalRepoNameLookup, moduleNameLookup);
depGraph,
canonicalRepoNameLookup,
moduleNameLookup,
abridgedModules,
extensionUsagesTable,
extensionUniqueNames);
}

/**
Expand All @@ -52,4 +65,46 @@ public static BazelModuleResolutionValue create(
* or modules with multiple-version overrides.
*/
public abstract ImmutableMap<String, ModuleKey> getModuleNameLookup();

/** All modules in the same order as {@link #getDepGraph}, but with limited information. */
public abstract ImmutableList<AbridgedModule> getAbridgedModules();

/**
* All module extension usages grouped by the extension's ID and the key of the module where this
* usage occurs. For each extension identifier ID, extensionUsagesTable[ID][moduleKey] is the
* ModuleExtensionUsage of ID in the module keyed by moduleKey.
*/
public abstract ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage>
getExtensionUsagesTable();

/**
* A mapping from the ID of a module extension to a unique string that serves as its "name". This
* is not the same as the extension's declared name, as the declared name is only unique within
* the .bzl file, whereas this unique name is guaranteed to be unique across the workspace.
*/
public abstract ImmutableMap<ModuleExtensionId, String> getExtensionUniqueNames();

/**
* Returns the full {@link RepositoryMapping} for the given module, including repos from Bazel
* module deps and module extensions.
*/
public final RepositoryMapping getFullRepoMapping(ModuleKey key) {
ImmutableMap.Builder<RepositoryName, RepositoryName> mapping = ImmutableMap.builder();
for (Map.Entry<ModuleExtensionId, ModuleExtensionUsage> e :
getExtensionUsagesTable().column(key).entrySet()) {
ModuleExtensionId extensionId = e.getKey();
ModuleExtensionUsage usage = e.getValue();
for (Map.Entry<String, String> entry : usage.getImports().entrySet()) {
String canonicalRepoName =
getExtensionUniqueNames().get(extensionId) + "." + entry.getValue();
mapping.put(
RepositoryName.createFromValidStrippedName(entry.getKey()),
RepositoryName.createFromValidStrippedName(canonicalRepoName));
}
}
return getDepGraph()
.get(key)
.getRepoMappingWithBazelDepsOnly()
.withAdditionalMappings(mapping.build());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ public abstract class Module {
*/
public abstract ModuleKey getKey();

public final String getCanonicalRepoName() {
return getKey().getCanonicalRepoName();
}

/**
* The compatibility level of the module, which essentially signifies the "major version" of the
* module in terms of SemVer.
Expand All @@ -75,49 +79,36 @@ public abstract class Module {
*/
public abstract ImmutableMap<String, ModuleKey> getDeps();

/**
* Used in {@link #getRepoMapping} to denote whether only repos from {@code bazel_dep}s should be
* returned, or repos from module extensions should also be returned.
*/
public enum WhichRepoMappings {
BAZEL_DEPS_ONLY,
WITH_MODULE_EXTENSIONS_TOO
}

/** Returns the {@link RepositoryMapping} for the repo corresponding to this module. */
public final RepositoryMapping getRepoMapping(WhichRepoMappings whichRepoMappings) {
static RepositoryMapping getRepoMappingWithBazelDepsOnly(
ModuleKey key, String name, ImmutableMap<String, ModuleKey> deps) {
ImmutableMap.Builder<RepositoryName, RepositoryName> mapping = ImmutableMap.builder();
// If this is the root module, then the main repository should be visible as `@`.
if (getKey().equals(ModuleKey.ROOT)) {
if (key.equals(ModuleKey.ROOT)) {
mapping.put(RepositoryName.MAIN, RepositoryName.MAIN);
}
// Every module should be able to reference itself as @<module name>.
// If this is the root module, this perfectly falls into @<module name> => @
if (!getName().isEmpty()) {
if (!name.isEmpty()) {
mapping.put(
RepositoryName.createFromValidStrippedName(getName()),
RepositoryName.createFromValidStrippedName(getKey().getCanonicalRepoName()));
RepositoryName.createFromValidStrippedName(name),
RepositoryName.createFromValidStrippedName(key.getCanonicalRepoName()));
}
for (Map.Entry<String, ModuleKey> dep : getDeps().entrySet()) {
for (Map.Entry<String, ModuleKey> dep : deps.entrySet()) {
// Special note: if `dep` is actually the root module, its ModuleKey would be ROOT whose
// canonicalRepoName is the empty string. This perfectly maps to the main repo ("@").
mapping.put(
RepositoryName.createFromValidStrippedName(dep.getKey()),
RepositoryName.createFromValidStrippedName(dep.getValue().getCanonicalRepoName()));
}
if (whichRepoMappings.equals(WhichRepoMappings.WITH_MODULE_EXTENSIONS_TOO)) {
for (ModuleExtensionUsage usage : getExtensionUsages()) {
for (Map.Entry<String, String> entry : usage.getImports().entrySet()) {
// TODO(wyv): work out a rigorous canonical repo name format (and potentially a shorter
// version when ambiguities aren't present).
String canonicalRepoName = usage.getExtensionName() + "." + entry.getValue();
mapping.put(
RepositoryName.createFromValidStrippedName(entry.getKey()),
RepositoryName.createFromValidStrippedName(canonicalRepoName));
}
}
}
return RepositoryMapping.create(mapping.build(), getKey().getCanonicalRepoName());
return RepositoryMapping.create(mapping.build(), key.getCanonicalRepoName());
}

/**
* Returns a {@link RepositoryMapping} with only Bazel module repos and no repos from module
* extensions. For the full mapping, see {@link BazelModuleResolutionValue#getFullRepoMappings}.
*/
public final RepositoryMapping getRepoMappingWithBazelDepsOnly() {
return getRepoMappingWithBazelDepsOnly(getKey(), getName(), getDeps());
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.auto.value.AutoValue;
import com.google.devtools.build.lib.cmdline.Label;

/** A unique identifier for a {@link ModuleExtension}. */
@AutoValue
public abstract class ModuleExtensionId {
public abstract Label getBzlFileLabel();

public abstract String getExtensionName();

public static ModuleExtensionId create(Label bzlFileLabel, String extensionName) {
return new AutoValue_ModuleExtensionId(bzlFileLabel, extensionName);
}
}
Loading

0 comments on commit 3aaddca

Please sign in to comment.