Compile the standard library on demand (#956)

* Compile the standard library on demand

* Review feedback

go/private/BUILD.sdk.bazel

@@ -3,59 +3,20 @@ load("@io_bazel_rules_go//go/private:rules/stdlib.bzl", "stdlib")
 package(default_visibility = [ "//visibility:public" ])
 
 filegroup(
-  name = "root_file",
-  srcs = ["ROOT"],
+  name = "host_sdk",
+  srcs = glob([
+    "bin/go*",
+    "src/**",
+    "pkg/**",
+  ]),
 )
 
 filegroup(
-  name = "go",
-  srcs = [ "bin/go{extension}" ],
-)
-
-filegroup(
-  name = "tools",
-  srcs = [":go"] + glob(["pkg/tool/**"]),
-)
-
-filegroup(
-    name = "srcs",
-    srcs = glob(["src/**"]),
-)
-
-cc_library(
-  name = "headers",
-  hdrs = glob(["pkg/include/*.h"]),
+  name = "host_tools",
+  srcs = select({
+    "@io_bazel_rules_go//go/platform:darwin_amd64": ["@local_config_cc//:cc_wrapper"],
+    "//conditions:default": None,
+  }),
 )
 
 exports_files(["packages.txt"])
-
-[
-  (
-    stdlib(
-        name = "stdlib_{}_{}".format(goos, goarch),
-        goos = goos,
-        goarch = goarch,
-        libs = glob(["pkg/{}_{}/**".format(goos, goarch)]),
-        cgo = True,
-    ),
-    stdlib(
-        name = "stdlib_{}_{}_pure".format(goos, goarch),
-        goos = goos,
-        goarch = goarch,
-        libs = glob(["pkg/{}_{}/**".format(goos, goarch)]),
-        cgo = False,
-    ),
-  )
-  for goos, goarch in [
-      ("darwin", "amd64"),
-      ("linux", "386"),
-      ("linux", "amd64"),
-      ("linux", "armv6l"),
-      ("windows", "386"),
-      ("windows", "amd64"),
-      ("freebsd", "386"),
-      ("freebsd", "amd64"),
-      ("linux", "ppc64le"),
-      ("linux", "s390x"),
-  ]
-]

go/private/actions/action.bzl

@@ -12,26 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-def action_with_go_env(ctx, go_toolchain, executable = None, command=None, arguments = [], inputs = [], **kwargs):
+def action_with_go_env(ctx, go_toolchain, stdlib, executable = None, command=None, arguments = [], inputs = [], **kwargs):
   if command:
     fail("You cannot run action_with_go_env with a 'command', only an 'executable'")
   args = [
-      "-go", go_toolchain.tools.go.path,
-      "-root_file", go_toolchain.stdlib.root_file.path,
-      "-goos", go_toolchain.stdlib.goos,
-      "-goarch", go_toolchain.stdlib.goarch,
-      "-cgo=" + ("1" if go_toolchain.stdlib.cgo else "0"),
+      "-go", stdlib.go.path,
+      "-root_file", stdlib.root_file.path,
+      "-goos", stdlib.goos,
+      "-goarch", stdlib.goarch,
+      "-cgo=" + ("1" if stdlib.cgo else "0"),
   ] + arguments
   ctx.action(
-      inputs = depset(inputs) + go_toolchain.data.tools + [go_toolchain.stdlib.root_file] + go_toolchain.stdlib.libs,
+      inputs = depset(inputs) + stdlib.files,
       executable = executable,
       arguments = args,
       **kwargs)
 
 def bootstrap_action(ctx, go_toolchain, inputs, outputs, mnemonic, arguments):
+  stdlib = go_toolchain.stdlib.cgo
   ctx.actions.run_shell(
-    inputs = inputs + go_toolchain.data.tools + go_toolchain.stdlib.libs,
+    inputs = inputs + stdlib.files,
     outputs = outputs,
     mnemonic = mnemonic,
-    command = "export GOROOT=$(pwd)/{} && {} {}".format(go_toolchain.stdlib.root_file.dirname, go_toolchain.tools.go.path, " ".join(arguments)),
+    command = "export GOROOT=$(pwd)/{} && {} {}".format(stdlib.root_file.dirname, stdlib.go.path, " ".join(arguments)),
   )

go/private/actions/asm.bzl

@@ -24,14 +24,14 @@ def emit_asm(ctx, go_toolchain,
 
   if source == None: fail("source is a required parameter")
   if out_obj == None: fail("out_obj is a required parameter")
-  includes = depset()
+  stdlib = go_toolchain.stdlib.get(ctx, go_toolchain)
+  includes = depset([stdlib.root_file.dirname + "/pkg/include"])
   includes += [f.dirname for f in hdrs]
-  includes += [f.dirname for f in go_toolchain.data.headers.cc.transitive_headers]
-  inputs = hdrs + list(go_toolchain.data.headers.cc.transitive_headers) + [source]
+  inputs = hdrs + stdlib.files + [source]
   asm_args = [source.path, "-o", out_obj.path]
   for inc in includes:
     asm_args += ["-I", inc]
-  action_with_go_env(ctx, go_toolchain,
+  action_with_go_env(ctx, go_toolchain, stdlib,
       inputs = list(inputs),
       outputs = [out_obj],
       mnemonic = "GoAsmCompile",

go/private/actions/compile.bzl

@@ -37,6 +37,8 @@ def emit_compile(ctx, go_toolchain,
   if sources == None: fail("sources is a required parameter")
   if out_lib == None: fail("out_lib is a required parameter")
 
+  stdlib = go_toolchain.stdlib.get(ctx, go_toolchain)
+
   # Add in any mode specific behaviours
   if mode == RACE_MODE:
     gc_goopts = gc_goopts + ("-race",)
@@ -61,7 +63,7 @@ def emit_compile(ctx, go_toolchain,
   if ctx.attr._go_toolchain_flags.compilation_mode == "debug":
     args.extend(["-N", "-l"])
   args.extend(cgo_sources)
-  action_with_go_env(ctx, go_toolchain,
+  action_with_go_env(ctx, go_toolchain, stdlib,
       inputs = list(inputs),
       outputs = [out_lib],
       mnemonic = "GoCompile",

go/private/actions/cover.bzl

@@ -19,6 +19,8 @@ load("@io_bazel_rules_go//go/private:actions/action.bzl",
 def emit_cover(ctx, go_toolchain,
                sources = []):
   """See go/toolchains.rst#cover for full documentation."""
+
+  stdlib = go_toolchain.stdlib.get(ctx, go_toolchain)
   outputs = []
   # TODO(linuxerwang): make the mode configurable.
   cover_vars = []
@@ -34,7 +36,7 @@ def emit_cover(ctx, go_toolchain,
     cover_vars += ["{}={}".format(cover_var,src.short_path)]
     out = ctx.new_file(cover_var + '.cover.go')
     outputs += [out]
-    action_with_go_env(ctx, go_toolchain,
+    action_with_go_env(ctx, go_toolchain, stdlib,
         inputs = [src],
         outputs = [out],
         mnemonic = "GoCover",

go/private/actions/link.bzl

@@ -37,15 +37,16 @@ def emit_link(ctx, go_toolchain,
   if library == None: fail("library is a required parameter")
   if executable == None: fail("executable is a required parameter")
 
+  stdlib = go_toolchain.stdlib.get(ctx, go_toolchain)
 
   config_strip = len(ctx.configuration.bin_dir.path) + 1
   pkg_depth = executable.dirname[config_strip:].count('/') + 1
 
   ld = None
   extldflags = []
-  if go_toolchain.external_linker:
-    ld = go_toolchain.external_linker.compiler_executable
-    extldflags = list(go_toolchain.external_linker.options)
+  if stdlib.cgo_tools:
+    ld = stdlib.cgo_tools.compiler_executable
+    extldflags = list(stdlib.cgo_tools.options)
   extldflags += ["-Wl,-rpath,$ORIGIN/" + ("../" * pkg_depth)]
 
   gc_linkopts, extldflags = _extract_extldflags(gc_linkopts, extldflags)
@@ -113,7 +114,7 @@ def emit_link(ctx, go_toolchain,
 
   link_args += ["--"] + link_opts
 
-  action_with_go_env(ctx, go_toolchain,
+  action_with_go_env(ctx, go_toolchain, stdlib,
       inputs = list(libs + cgo_deps +
                 go_toolchain.data.crosstool + stamp_inputs),
       outputs = [executable],

go/private/actions/pack.bzl

@@ -26,6 +26,8 @@ def emit_pack(ctx, go_toolchain,
   if in_lib == None: fail("in_lib is a required parameter")
   if out_lib == None: fail("out_lib is a required parameter")
 
+  stdlib = go_toolchain.stdlib.get(ctx, go_toolchain)
+
   inputs = [in_lib]
 
   arguments = [
@@ -40,7 +42,7 @@ def emit_pack(ctx, go_toolchain,
     inputs.append(archive)
     arguments.extend(["-arc", archive.path])
 
-  action_with_go_env(ctx, go_toolchain,
+  action_with_go_env(ctx, go_toolchain, stdlib,
       inputs = inputs,
       outputs = [out_lib],
       mnemonic = "GoPack",

go/private/go_toolchain.bzl

@@ -23,11 +23,26 @@ load("@io_bazel_rules_go//go/private:actions/link.bzl", "emit_link", "bootstrap_
 load("@io_bazel_rules_go//go/private:actions/pack.bzl", "emit_pack")
 load("@io_bazel_rules_go//go/private:providers.bzl", "GoStdLib")
 
+def _get_stdlib(ctx, go_toolchain):
+  if "pure" in ctx.features and "race" in ctx.features:
+    return go_toolchain.stdlib.pure_race
+  elif "pure" in ctx.features:
+    return go_toolchain.stdlib.pure
+  elif "race" in ctx.features:
+    return go_toolchain.stdlib.cgo_race
+  else:
+    return go_toolchain.stdlib.cgo
 
 def _go_toolchain_impl(ctx):
   return [platform_common.ToolchainInfo(
       name = ctx.label.name,
-      stdlib = ctx.attr._stdlib[GoStdLib],
+      stdlib = struct(
+          cgo = ctx.attr._stdlib_cgo[GoStdLib],
+          pure = ctx.attr._stdlib_pure[GoStdLib],
+          cgo_race = ctx.attr._stdlib_cgo_race[GoStdLib],
+          pure_race = ctx.attr._stdlib_pure_race[GoStdLib],
+          get = _get_stdlib,
+      ),
       actions = struct(
           asm = emit_asm,
           binary = emit_binary,
@@ -38,7 +53,6 @@ def _go_toolchain_impl(ctx):
           pack = emit_pack,
       ),
       tools = struct(
-          go = ctx.executable._go,
           asm = ctx.executable._asm,
           compile = ctx.executable._compile,
           pack = ctx.executable._pack,
@@ -53,22 +67,22 @@ def _go_toolchain_impl(ctx):
           link_cgo = ctx.attr.cgo_link_flags,
       ),
       data = struct(
-          tools = ctx.files._tools,
-          stdlib = ctx.files._stdlib,
-          headers = ctx.attr._headers,
           crosstool = ctx.files._crosstool,
           package_list = ctx.file._package_list,
       ),
-      external_linker = ctx.attr._external_linker,
   )]
 
-def _stdlib(goos, goarch):
-  return Label("@go_sdk//:stdlib_{}_{}".format(goos, goarch))
+def _stdlib_cgo(goos, goarch):
+  return Label("@go_stdlib_{}_{}_cgo".format(goos, goarch))
+
+def _stdlib_pure(goos, goarch):
+  return Label("@go_stdlib_{}_{}_pure".format(goos, goarch))
 
-def _get_linker():
-  # TODO: return None if there is no cpp fragment available
-  # This is not possible right now, we need a new bazel feature
-  return Label("//go/toolchain:external_linker")
+def _stdlib_cgo_race(goos, goarch):
+  return Label("@go_stdlib_{}_{}_cgo_race".format(goos, goarch))
+
+def _stdlib_pure_race(goos, goarch):
+  return Label("@go_stdlib_{}_{}_pure_race".format(goos, goarch))
 
 def _asm(bootstrap):
   if bootstrap:
@@ -124,13 +138,12 @@ _go_toolchain = rule(
         "_test_generator": attr.label(allow_files = True, single_file = True, executable = True, cfg = "host", default = _test_generator),
         "_cover": attr.label(allow_files = True, single_file = True, executable = True, cfg = "host", default = _cover),
         # Hidden internal attributes
-        "_go": attr.label(allow_files = True, single_file = True, executable = True, cfg = "host", default="@go_sdk//:go"),
-        "_tools": attr.label(allow_files = True, default = "@go_sdk//:tools"),
-        "_stdlib": attr.label(allow_files = True, default = _stdlib),
-        "_headers": attr.label(default="@go_sdk//:headers"),
+        "_stdlib_cgo": attr.label(allow_files = True, default = _stdlib_cgo),
+        "_stdlib_pure": attr.label(allow_files = True, default = _stdlib_pure),
+        "_stdlib_cgo_race": attr.label(allow_files = True, default = _stdlib_cgo_race),
+        "_stdlib_pure_race": attr.label(allow_files = True, default = _stdlib_pure_race),
         "_crosstool": attr.label(default=Label("//tools/defaults:crosstool")),
         "_package_list": attr.label(allow_files = True, single_file = True, default="@go_sdk//:packages.txt"),
-        "_external_linker": attr.label(default=_get_linker),
     },
 )
 
@@ -201,25 +214,3 @@ go_toolchain_flags = rule(
         "strip": attr.string(mandatory=True),
     },
 )
-
-def _external_linker_impl(ctx):
-  cpp = ctx.fragments.cpp
-  features = ctx.features
-  options = (cpp.compiler_options(features) +
-        cpp.unfiltered_compiler_options(features) +
-        cpp.link_options +
-        cpp.mostly_static_link_options(features, False))
-  return struct(
-      compiler_executable = cpp.compiler_executable,
-      options = options,
-      c_options = cpp.c_options,
-  )
-
-_external_linker = rule(
-    _external_linker_impl,
-    attrs = {},
-    fragments = ["cpp"],
-)
-
-def external_linker():
-    _external_linker(name="external_linker")

go/private/repositories.bzl

@@ -16,6 +16,7 @@
 
 load("@io_bazel_rules_go//go/private:repository_tools.bzl", "go_repository_tools")
 load("@io_bazel_rules_go//go/private:go_repository.bzl", "go_repository")
+load('@io_bazel_rules_go//go/private:rules/stdlib.bzl', "STDLIB_GOOS_GOARCH", "go_stdlib")
 load('@io_bazel_rules_go//go/toolchain:toolchains.bzl', "go_register_toolchains")
 
 def go_rules_dependencies():
@@ -39,11 +40,40 @@ def go_rules_dependencies():
       type = "zip",
   )
 
+  for goos, goarch in STDLIB_GOOS_GOARCH:
+    _maybe(go_stdlib,
+        name = "go_stdlib_{}_{}_cgo".format(goos, goarch),
+        goos = goos,
+        goarch = goarch,
+        race = False,
+        cgo = True,
+    )
+    _maybe(go_stdlib,
+        name = "go_stdlib_{}_{}_pure".format(goos, goarch),
+        goos = goos,
+        goarch = goarch,
+        race = False,
+        cgo = False,
+    )
+    _maybe(go_stdlib,
+        name = "go_stdlib_{}_{}_cgo_race".format(goos, goarch),
+        goos = goos,
+        goarch = goarch,
+        race = True,
+        cgo = True,
+    )
+    _maybe(go_stdlib,
+        name = "go_stdlib_{}_{}_pure_race".format(goos, goarch),
+        goos = goos,
+        goarch = goarch,
+        race = True,
+        cgo = False,
+    )
+
   _maybe(go_repository_tools,
       name = "io_bazel_rules_go_repository_tools",
   )
 
-
   # Proto dependancies
   _maybe(go_repository,
       name = "com_github_golang_protobuf",