Skip to content

Commit

Permalink
Builtins injection: Rename _internal to _builtins and add functionality
Browse files Browse the repository at this point in the history
This object is used in @_builtins .bzl files but is not accessible to user code.

The `toplevel` and `native` fields give access to the *native* (pre-injected) values of symbols whose post-injected values are available to user .bzl files. For instance, -`_builtins.toplevel.CcInfo` in @_builtins code gives the original CcInfo definition from the Java code, even if `CcInfo` in a regular .bzl file refers to an injected value. (To avoid ambiguity, `CcInfo` itself is not a valid top-level symbol for @_builtins .bzl files.)

The `internal` field contains any value registered via ConfiguredRuleClassProvider.Builder#addStarlarkBuiltinsInternal().

The `getFlag()` method can retrieve the values of StarlarkSemantics flags. Because of how flags are stored, it requires that a default value be given.

Work toward bazelbuild#11437.

PiperOrigin-RevId: 356550108
  • Loading branch information
brandjon authored and copybara-github committed Feb 9, 2021
1 parent db24c6d commit be96ade
Show file tree
Hide file tree
Showing 10 changed files with 450 additions and 117 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,9 @@ public static class Builder implements RuleDefinitionEnvironment {
OptionsDiffPredicate.ALWAYS_INVALIDATE;
private PrerequisiteValidator prerequisiteValidator;
private final ImmutableList.Builder<Bootstrap> starlarkBootstraps = ImmutableList.builder();
private ImmutableMap.Builder<String, Object> starlarkAccessibleTopLevels =
private final ImmutableMap.Builder<String, Object> starlarkAccessibleTopLevels =
ImmutableMap.builder();
private final ImmutableMap.Builder<String, Object> starlarkBuiltinsInternals =
ImmutableMap.builder();
private final ImmutableList.Builder<SymlinkDefinition> symlinkDefinitions =
ImmutableList.builder();
Expand Down Expand Up @@ -261,6 +263,11 @@ public Builder addStarlarkAccessibleTopLevels(String name, Object object) {
return this;
}

public Builder addStarlarkBuiltinsInternal(String name, Object object) {
this.starlarkBuiltinsInternals.put(name, object);
return this;
}

public Builder addSymlinkDefinition(SymlinkDefinition symlinkDefinition) {
this.symlinkDefinitions.add(symlinkDefinition);
return this;
Expand Down Expand Up @@ -497,6 +504,7 @@ public ConfiguredRuleClassProvider build() {
shouldInvalidateCacheForOptionDiff,
prerequisiteValidator,
starlarkAccessibleTopLevels.build(),
starlarkBuiltinsInternals.build(),
starlarkBootstraps.build(),
symlinkDefinitions.build(),
ImmutableSet.copyOf(reservedActionMnemonics),
Expand Down Expand Up @@ -586,6 +594,8 @@ public String getToolsRepository() {

private final ImmutableMap<String, Object> nativeRuleSpecificBindings;

private final ImmutableMap<String, Object> starlarkBuiltinsInternals;

private final ImmutableMap<String, Object> environment;

private final ImmutableList<SymlinkDefinition> symlinkDefinitions;
Expand Down Expand Up @@ -619,7 +629,8 @@ private ConfiguredRuleClassProvider(
PatchTransition toolchainTaggedTrimmingTransition,
OptionsDiffPredicate shouldInvalidateCacheForOptionDiff,
PrerequisiteValidator prerequisiteValidator,
ImmutableMap<String, Object> starlarkAccessibleJavaClasses,
ImmutableMap<String, Object> starlarkAccessibleTopLevels,
ImmutableMap<String, Object> starlarkBuiltinsInternals,
ImmutableList<Bootstrap> starlarkBootstraps,
ImmutableList<SymlinkDefinition> symlinkDefinitions,
ImmutableSet<String> reservedActionMnemonics,
Expand All @@ -646,7 +657,8 @@ private ConfiguredRuleClassProvider(
this.shouldInvalidateCacheForOptionDiff = shouldInvalidateCacheForOptionDiff;
this.prerequisiteValidator = prerequisiteValidator;
this.nativeRuleSpecificBindings =
createNativeRuleSpecificBindings(starlarkAccessibleJavaClasses, starlarkBootstraps);
createNativeRuleSpecificBindings(starlarkAccessibleTopLevels, starlarkBootstraps);
this.starlarkBuiltinsInternals = starlarkBuiltinsInternals;
this.environment = createEnvironment(nativeRuleSpecificBindings);
this.symlinkDefinitions = symlinkDefinitions;
this.reservedActionMnemonics = reservedActionMnemonics;
Expand Down Expand Up @@ -844,6 +856,11 @@ public ImmutableMap<String, Object> getNativeRuleSpecificBindings() {
return nativeRuleSpecificBindings;
}

@Override
public ImmutableMap<String, Object> getStarlarkBuiltinsInternals() {
return starlarkBuiltinsInternals;
}

@Override
public ImmutableMap<String, Object> getEnvironment() {
return environment;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,16 @@

package com.google.devtools.build.lib.packages;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.starlark.java.eval.FlagGuardedValue;
import net.starlark.java.eval.Starlark;

// TODO(adonovan): move skyframe.PackageFunction into lib.packages so we needn't expose this and
// the other env-building functions.
/**
* This class encapsulates knowledge of how to set up the Starlark environment for BUILD, WORKSPACE,
* and bzl file evaluation, including the top-level predeclared symbols, the {@code native} module,
Expand All @@ -31,12 +35,6 @@
* result of (1) is cached by an instance of this class. (2) is obtained using helper methods on
* this class, and cached in {@link StarlarkBuiltinsValue}.
*/
// TODO(#11437): To deal with StarlarkSemantics inspection via _builtins.flags, we can make it a
// function (not a struct field) that accesses the StarlarkThread. To deal with flag-guarded
// top-level symbols exposed under _builtins.toplevel, we can simply make them all accessible
// unconditionally, under the principle that the flag migrator should be responsible for deleting
// all uses from @_builtins code, and @_builtins code can be trusted to use flag-guarded features
// responsibly.
public final class BazelStarlarkEnvironment {

// TODO(#11954): Eventually the BUILD and WORKSPACE bzl dialects should converge. Right now they
Expand Down Expand Up @@ -73,7 +71,9 @@ public final class BazelStarlarkEnvironment {
this.uninjectedBuildBzlEnv =
createUninjectedBuildBzlEnv(ruleClassProvider, uninjectedBuildBzlNativeBindings);
this.workspaceBzlEnv = createWorkspaceBzlEnv(ruleClassProvider, workspaceBzlNativeBindings);
this.builtinsBzlEnv = createBuiltinsBzlEnv(ruleClassProvider);
this.builtinsBzlEnv =
createBuiltinsBzlEnv(
ruleClassProvider, uninjectedBuildBzlNativeBindings, uninjectedBuildBzlEnv);
this.uninjectedBuildEnv =
createUninjectedBuildEnv(ruleFunctions, packageFunction, environmentExtensions);
}
Expand Down Expand Up @@ -194,14 +194,43 @@ private static ImmutableMap<String, Object> createWorkspaceBzlEnv(
}

private static ImmutableMap<String, Object> createBuiltinsBzlEnv(
RuleClassProvider ruleClassProvider) {
RuleClassProvider ruleClassProvider,
ImmutableMap<String, Object> uninjectedBuildBzlNativeBindings,
ImmutableMap<String, Object> uninjectedBuildBzlEnv) {
Map<String, Object> env = new HashMap<>();
env.putAll(ruleClassProvider.getEnvironment());

// Clear out rule-specific symbols like CcInfo.
env.keySet().removeAll(ruleClassProvider.getNativeRuleSpecificBindings().keySet());

env.put("_internal", InternalModule.INSTANCE);
// For _builtins.toplevel, replace all FlagGuardedValues with the underlying value;
// StarlarkSemantics flags do not affect @_builtins.
//
// We do this because otherwise we'd need to differentiate the _builtins.toplevel object (and
// therefore the @_builtins environment) based on StarlarkSemantics. That seems unnecessary.
// Instead we trust @_builtins to not misuse flag-guarded features, same as native code.
//
// If foo is flag-guarded (either experimental or incompatible), it is unconditionally visible
// as _builtins.toplevel.foo. It is legal to list it in exported_toplevels unconditionally, but
// the flag still controls whether the symbol is actually visible to user code.
Map<String, Object> unwrappedBuildBzlSymbols = new HashMap<>();
for (Map.Entry<String, Object> entry : uninjectedBuildBzlEnv.entrySet()) {
Object symbol = entry.getValue();
if (symbol instanceof FlagGuardedValue) {
symbol = ((FlagGuardedValue) symbol).getObject();
}
unwrappedBuildBzlSymbols.put(entry.getKey(), symbol);
}

Object builtinsModule =
new BuiltinsInternalModule(
createNativeModule(uninjectedBuildBzlNativeBindings),
// createNativeModule() is good enough for the "toplevel" and "internal" objects too.
createNativeModule(unwrappedBuildBzlSymbols),
createNativeModule(ruleClassProvider.getStarlarkBuiltinsInternals()));
Object conflictingValue = env.put("_builtins", builtinsModule);
Preconditions.checkState(
conflictingValue == null, "'_builtins' name is reserved for builtins injection");

return ImmutableMap.copyOf(env);
}
Expand All @@ -220,16 +249,6 @@ private static ImmutableMap<String, Object> createBuiltinsBzlEnv(
public ImmutableMap<String, Object> createBuildBzlEnvUsingInjection(
Map<String, Object> injectedToplevels, Map<String, Object> injectedRules)
throws InjectionException {
// TODO(#11437): Builtins injection should take into account StarlarkSemantics and
// FlagGuardedValues. If a builtin is disabled by a flag, we can either:
//
// 1) Treat it as if it doesn't exist for the purposes of injection. In this case it's an
// error to attempt to inject it, so exports.bzl is required to explicitly check the flag's
// value (via the _internal module) before exporting it.
//
// 2) Allow it to be exported and automatically suppress/omit it from the final environment,
// effectively rewrapping the injected builtin in the FlagGuardedValue.

// Determine top-level symbols.
Map<String, Object> env = new HashMap<>();
env.putAll(uninjectedBuildBzlEnv);
Expand Down Expand Up @@ -276,8 +295,6 @@ public ImmutableMap<String, Object> createBuildBzlEnvUsingInjection(
* overridden in this manner, not generic built-ins such as {@code package} or {@code glob}.
* Throws InjectionException if these conditions are not met.
*/
// TODO(adonovan): move skyframe.PackageFunction into lib.packages so we needn't expose this and
// the other env-building functions.
public ImmutableMap<String, Object> createBuildEnvUsingInjection(
Map<String, Object> injectedRules) throws InjectionException {
HashMap<String, Object> env = new HashMap<>(uninjectedBuildEnv);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.packages;

import com.google.devtools.build.docgen.annot.DocCategory;
import net.starlark.java.annot.Param;
import net.starlark.java.annot.StarlarkBuiltin;
import net.starlark.java.annot.StarlarkMethod;
import net.starlark.java.eval.Printer;
import net.starlark.java.eval.Starlark;
import net.starlark.java.eval.StarlarkThread;
import net.starlark.java.eval.StarlarkValue;

// TODO(#11437): Note that if Stardoc's current design were to be long-lived, we'd want to factor
// out an API into starlarkbuildapi. As it is that almost certainly won't be necessary.
/**
* The {@code _builtins} Starlark object, visible only to {@code @_builtins} .bzl files, supporting
* access to internal APIs.
*/
@StarlarkBuiltin(
name = "_builtins",
category = DocCategory.BUILTIN,
documented = false,
doc =
"A module accessible only to @_builtins .bzls, that permits access to the original "
+ "(uninjected) native builtins, as well as internal-only symbols not accessible to "
+ "users.")
public class BuiltinsInternalModule implements StarlarkValue {

// _builtins.native
private final Object uninjectedNativeObject;
// _builtins.toplevel
private final Object uninjectedToplevelObject;
// _builtins.internal
private final Object internalObject;

public BuiltinsInternalModule(
Object uninjectedNativeObject, Object uninjectedToplevelObject, Object internalObject) {
this.uninjectedNativeObject = uninjectedNativeObject;
this.uninjectedToplevelObject = uninjectedToplevelObject;
this.internalObject = internalObject;
}

@Override
public void repr(Printer printer) {
printer.append("<_builtins module>");
}

@Override
public boolean isImmutable() {
return true;
}

@StarlarkMethod(
name = "native",
doc =
"A view of the <code>native</code> object as it would exist if builtins injection were"
+ " disabled. For example, if builtins injection provides a Starlark definition for"
+ " <code>cc_library</code> in <code>exported_rules</code>, then"
+ " <code>native.cc_library</code> in a user .bzl file would refer to that"
+ " definition, but <code>_builtins.native.cc_library</code> in a"
+ " <code>@_builtins</code> .bzl file would still be the one defined in Java code."
+ " (Note that for clarity and to avoid a conceptual cycle, the regular top-level"
+ " <code>native</code> object is not defined for <code>@_builtins</code> .bzl"
+ " files.)",
documented = false,
structField = true)
public Object getUninjectedNativeObject() {
return uninjectedNativeObject;
}

@StarlarkMethod(
name = "toplevel",
doc =
"A view of the top-level .bzl symbols that would exist if builtins injection were"
+ " disabled; analogous to <code>_builtins.native</code>. For example, if builtins"
+ " injection provides a Starlark definition for <code>CcInfo</code> in"
+ " <code>exported_toplevels</code>, then <code>_builtins.toplevel.CcInfo</code>"
+ " refers to the original Java definition, not the Starlark one. (Just as for"
+ " <code>_builtins.native</code>, the top-level <code>CcInfo</code> symbol is not"
+ " available to <code>@_builtins</code> .bzl files.)",
documented = false,
structField = true)
public Object getUninjectedToplevelObject() {
return uninjectedToplevelObject;
}

@StarlarkMethod(
name = "internal",
doc =
"A view of symbols that were registered (via the Java method"
+ "<code>ConfiguredRuleClassProvider#addStarlarkBuiltinsInternal</code>) to be made"
+ " available to <code>@_builtins</code> code but not necessarily user code.",
documented = false,
structField = true)
public Object getInternalObject() {
return internalObject;
}

@StarlarkMethod(
name = "get_flag",
doc =
"Returns the value of a <code>StarlarkSemantics</code> flag, or a default value if it"
+ " could not be retrieved (either because the flag does not exist or because it was"
+ " not assigned an explicit value). Fails if the flag value exists but is not a"
+ " Starlark value.",
documented = false,
parameters = {
@Param(name = "name", doc = "Name of the flag, without the leading dashes"),
/*
* Because of the way flag values are stored in StarlarkSemantics, we cannot retrieve a flag
* whose value was not explicitly set, nor can we programmatically determine its default
* value. This parameter is essentially a hack to avoid a slightly costlier refactoring of
* StarlarkSemantics and BuildLanguageOptions. If the value passed in here by the caller
* differs from the true default value of the flag, you could end up in a situation where
* the semantics of some @_builtins code varies depending on whether a flag was set to its
* default value implicitly or explicitly.
*/
@Param(
name = "default",
doc =
"Value to return if flag was not set or does not exist. This should always be set"
+ " to the same value as the flag's default value.")
},
useStarlarkThread = true)
public Object getFlag(String name, Object defaultValue, StarlarkThread thread) {
Object value = thread.getSemantics().getGeneric(name, defaultValue);
return Starlark.fromJava(value, thread.mutability());
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,15 @@ void setStarlarkThreadContext(
*/
ImmutableMap<String, Object> getNativeRuleSpecificBindings();

/**
* Returns the set of symbols to be made available to {@code @_builtins} .bzl files under the
* _builtins.internal object.
*
* <p>These symbols are not exposed to user .bzl code and do not constitute a public or stable API
* (unless exposed through another means).
*/
ImmutableMap<String, Object> getStarlarkBuiltinsInternals();

/**
* Returns the Starlark builtins registered with this RuleClassProvider.
*
Expand Down
Loading

0 comments on commit be96ade

Please sign in to comment.