buildkite: start testing the tldk network plugin

.buildkite/pipeline.yaml

@@ -80,6 +80,12 @@ steps:
       GLIBC_TUNABLES: glibc.pthread.rseq=0
     label: ":fire: Smoke race tests"
     command: make smoke-race-tests
+  - <<: *common
+    <<: *source_test
+    label: ":speedboat: Compile runsc-plugin-stack (AMD64)"
+    command: make runsc-plugin-stack
+    agents:
+      arch: "amd64"
 
   # Build runsc and pkg (presubmits only).
   - <<: *common
@@ -362,6 +368,14 @@ steps:
       <<: *platform_specific_agents
       <<: *ubuntu_agents
       arch: "amd64"
+  - <<: *common
+    <<: *docker
+    <<: *source_test
+    label: ":satellite: gVisor network plugin tests"
+    command: make plugin-network-tests
+    agents:
+      <<: *kvm_agents
+      arch: "amd64"
   - <<: *common
     <<: *source_test
     label: ":coffee: Do tests"

Makefile

@@ -111,6 +111,7 @@ else
 RUNTIME     ?= $(BRANCH_NAME)
 endif
 RUNTIME_DIR     ?= $(shell dirname $(shell mktemp -u))/$(RUNTIME)
+RUNSC_TARGET    ?= //runsc
 RUNTIME_BIN     ?= $(RUNTIME_DIR)/runsc
 RUNTIME_LOG_DIR ?= $(RUNTIME_DIR)/logs
 RUNTIME_LOGS    ?= $(RUNTIME_LOG_DIR)/runsc.log.%TEST%.%TIMESTAMP%.%COMMAND%
@@ -127,7 +128,7 @@ endif
 $(RUNTIME_BIN): # See below.
 	@mkdir -p "$(RUNTIME_DIR)"
 ifeq (,$(STAGED_BINARIES))
-	@$(call copy,//runsc,$(RUNTIME_BIN))
+	@$(call copy,$(RUNSC_TARGET),$(RUNTIME_BIN))
 else
 	gsutil cat "${STAGED_BINARIES}" | \
 	  tar -C "$(RUNTIME_DIR)" -zxvf - runsc && \
@@ -200,6 +201,10 @@ runsc: ## Builds the runsc binary.
 	@$(call build,-c opt //runsc)
 .PHONY: runsc
 
+runsc-plugin-stack:
+	@$(call build,-c opt $(PLUGIN_STACK_FLAGS) //runsc:runsc-plugin-stack)
+.PHONY: runsc-plugin-stack
+
 debian: ## Builds the debian packages.
 	@$(call build,-c opt //debian:debian)
 .PHONY: debian
@@ -342,6 +347,12 @@ docker-tests: load-basic $(RUNTIME_BIN)
 	@$(call test_runtime,$(RUNTIME),$(INTEGRATION_TARGETS) //test/e2e:integration_runtime_test)
 .PHONY: docker-tests
 
+plugin-network-tests: load-basic $(RUNTIME_BIN)
+	@$(call install_runtime,$(RUNTIME),--network=plugin)
+	@$(call test_runtime,$(RUNTIME), --test_arg=-test.run=ConnectToSelf $(INTEGRATION_TARGETS))
+
+plugin-network-tests: RUNSC_TARGET=--config plugin-tldk //runsc:runsc-plugin-stack
+
 overlay-tests: load-basic $(RUNTIME_BIN)
 	@$(call install_runtime,$(RUNTIME),--overlay2=all:dir=/tmp)
 	@$(call install_runtime,$(RUNTIME)-docker,--net-raw --overlay2=all:dir=/tmp)

WORKSPACE

@@ -199,6 +199,7 @@ register_toolchains("//:cc_toolchain_k8", "//:cc_toolchain_aarch64")
 
 # Load LLVM dependencies.
 LLVM_COMMIT = "926f85db98aae66ab8f57b9981f47ddddb868c51"
+
 LLVM_SHA256 = "c78c94b2a03b2cf6ef1ba035c31a6f1b0bb7913da8af5aa8d5c2061f6499d589"
 
 http_archive(
@@ -239,12 +240,12 @@ maybe(
 
 # Load other C++ dependencies.
 http_archive(
-  name = "nlohmann_json",
-  sha256 = "ba6e7817353793d13e5214ed819ea5b0defc0ffb2a348f4e34b10ac6f1c50154",
-  strip_prefix = "json-960b763ecd144f156d05ec61f577b04107290137",
-  urls = [
-    "https://github.com/nlohmann/json/archive/960b763ecd144f156d05ec61f577b04107290137.tar.gz"
-  ]
+    name = "nlohmann_json",
+    sha256 = "ba6e7817353793d13e5214ed819ea5b0defc0ffb2a348f4e34b10ac6f1c50154",
+    strip_prefix = "json-960b763ecd144f156d05ec61f577b04107290137",
+    urls = [
+        "https://github.com/nlohmann/json/archive/960b763ecd144f156d05ec61f577b04107290137.tar.gz",
+    ],
 )
 
 http_archive(
@@ -3371,6 +3372,6 @@ go_repository(
 
 new_local_repository(
     name = "libpluginstack",
-    path = "tools/plugin-stack",
     build_file = "tools/plugin-stack/plugin-stack.BUILD",
+    path = "external/tools/plugin-stack",
 )

tools/plugin-stack/plugin-stack.BUILD → external/tools/plugin-stack/plugin-stack.BUILD

@@ -6,12 +6,12 @@ config_setting(
 genrule(
     name = "pluginstack_genrule",
     outs = ["libpluginstack.a"],
-    local = 1,
     cmd = select({
         # Support IVB and later machines.
-        ":plugin_tldk_condition": "git clone git@github.com:alipay/tldk.git; cd tldk; git checkout 9efb0dacb67da1da62ca78785e8cffb0c5a82785; make -j 1 DPDK_MACHINE=ivb EXTRA_CFLAGS='-g -O3 -fPIC -fno-omit-frame-pointer -DLOOK_ASIDE_BACKEND -Wno-error' all; cd ..; cp -f tldk/libtldk.a $(RULEDIR)/libpluginstack.a",
+        ":plugin_tldk_condition": "git clone https://github.com/alipay/tldk.git; cd tldk; git checkout cec8ff773c2ee609a1fcbc389aecb4dbb4e3bb88; make -j 1 DPDK_MACHINE=ivb EXTRA_CFLAGS='-g -O3 -fPIC -fno-omit-frame-pointer -DLOOK_ASIDE_BACKEND -Wno-error -Wno-use-after-free' all; cd ..; cp -f tldk/libtldk.a $(RULEDIR)/libpluginstack.a",
         "//conditions:default": "",
     }),
+    local = 1,
     visibility = ["//visibility:public"],
 )
 

images/default/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y curl gnupg2 git \
         software-properties-common \
         pkg-config libffi-dev patch diffutils libssl-dev iptables kmod \
         clang crossbuild-essential-amd64 erofs-utils busybox-static libbpf-dev \
-        iproute2 netcat
+        iproute2 netcat libnuma-dev
 
 # This package is needed to build eBPF on amd64, but not on arm64 where it
 # doesn't exist.

runsc/sandbox/sandbox.go

@@ -852,11 +852,12 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 	if !conf.TestOnlyAllowRunAsCurrentUserWithoutChroot {
 		// Setting cmd.Env = nil causes cmd to inherit the current process's env.
 		cmd.Env = []string{}
-		// runsc-race with glibc needs to disable rseq.
-		glibcTunables := os.Getenv("GLIBC_TUNABLES")
-		if glibcTunables != "" {
-			cmd.Env = append(cmd.Env, fmt.Sprintf("GLIBC_TUNABLES=%s", glibcTunables))
-		}
+	}
+	if config.CgoEnabled {
+		// Platforms that use stub processes are not compatible with
+		// the glibc rseq, because they unmap everything from a process
+		// address space.
+		cmd.Env = append(cmd.Env, "GLIBC_TUNABLES=glibc.pthread.rseq=0")
 	}
 
 	// If there is a gofer, sends all socket ends to the sandbox.

test/root/runsc_test.go

@@ -172,7 +172,7 @@ func TestSandboxProcessEnv(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(got) != 0 {
-		t.Errorf("sandbox process's environment is not empty: got %s", string(got))
+	if len(got) != 0 && string(got) != "GLIBC_TUNABLES=glibc.pthread.rseq=0\x00" {
+		t.Errorf("sandbox process's environment is not empty: got %s (%v)", string(got), got)
 	}
 }

tools/bazel.mk

@@ -48,6 +48,7 @@ BRANCH_NAME := $(shell (git branch --show-current 2>/dev/null || \
   xargs -n 1 basename 2>/dev/null)
 BUILD_ROOTS := bazel-bin/ bazel-out/
 RACE_FLAGS := --config=race
+PLUGIN_STACK_FLAGS := --config=plugin-tldk
 
 # Bazel container configuration (see below).
 USER := $(shell whoami)