From bda0b6204a0899550b73ccc56256e029d4255628 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 7 May 2020 15:37:47 -0400 Subject: [PATCH 01/10] build: add support for pip3_import from rules_python. Provide an Envoy build system styled way of loading arbitrary pip3 packages as py_{library,binary} dependencies. Part of https://github.com/envoyproxy/envoy/issues/11085 (to fetch PyYAML). Risk level: Low (build system) Testing: Manual validation that the test utility loads PyYAML at the correct version (different to my system version). Signed-off-by: Harvey Tuch --- WORKSPACE | 4 ++++ bazel/dependency_imports.bzl | 3 +++ bazel/repositories.bzl | 4 ++-- bazel/repositories_extra.bzl | 16 ++++++++++++++++ bazel/repository_locations.bzl | 7 +++++-- tools/config_validation/BUILD | 10 ++++++++++ tools/config_validation/requirements.txt | 1 + .../config_validation/validate_yaml_fragment.py | 3 +++ 8 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 bazel/repositories_extra.bzl create mode 100644 tools/config_validation/BUILD create mode 100644 tools/config_validation/requirements.txt create mode 100644 tools/config_validation/validate_yaml_fragment.py diff --git a/WORKSPACE b/WORKSPACE index ef120bc53d4f..a96cba501302 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -12,6 +12,10 @@ load("//bazel:repositories.bzl", "envoy_dependencies") envoy_dependencies() +load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra") + +envoy_dependencies_extra() + load("//bazel:dependency_imports.bzl", "envoy_dependency_imports") envoy_dependency_imports() diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 051923e315a4..2385a98ff25b 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -4,6 +4,7 @@ load("@envoy_build_tools//toolchains:rbe_toolchains_config.bzl", "rbe_toolchains load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") load("@upb//bazel:repository_defs.bzl", upb_bazel_version_repository = "bazel_version_repository") +load("@config_validation//:requirements.bzl", config_validation_pip_install = "pip_install") # go version for rules_go GO_VERSION = "1.13.5" @@ -38,3 +39,5 @@ def envoy_dependency_imports(go_version = GO_VERSION): sum = "h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=", version = "v0.3.0", ) + + config_validation_pip_install() diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 18230f79d954..73e52e3eb7fe 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -65,9 +65,9 @@ _default_envoy_build_config = repository_rule( }, ) -# Python dependencies. If these become non-trivial, we might be better off using a virtualenv to -# wrap them, but for now we can treat them as first-class Bazel. +# Python dependencies. def _python_deps(): + # TODO(htuch): convert these to pip3_import. _repository_impl( name = "com_github_pallets_markupsafe", build_file = "@envoy//bazel/external:markupsafe.BUILD", diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl new file mode 100644 index 000000000000..277b6b6736c8 --- /dev/null +++ b/bazel/repositories_extra.bzl @@ -0,0 +1,16 @@ +load("@rules_python//python:repositories.bzl", "py_repositories") +load("@rules_python//python:pip.bzl", "pip3_import", "pip_repositories") + +# Python dependencies. +def _python_deps(): + py_repositories() + pip_repositories() + + pip3_import( + name = "config_validation", + requirements = "//tools/config_validation:requirements.txt", + ) + +# Envoy deps that rely on a first stage of dependency loading in envoy_dependencies(). +def envoy_dependencies_extra(): + _python_deps() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 9561aae50d6f..2a83b77b50f7 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -365,8 +365,11 @@ DEPENDENCY_REPOSITORIES = dict( use_category = ["build"], ), rules_python = dict( - sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161", - urls = ["https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz"], + sha256 = "76a8fd4e7eca2a3590f816958faa0d83c9b2ce9c32634c5c375bcccf161d3bb5", + strip_prefix = "rules_python-a0fbf98d4e3a232144df4d0d80b577c7a693b570", + # 2020-04-09 + # TODO(htuch): revert back to a point releases when pip3_import appears. + urls = ["https://github.com/bazelbuild/rules_python/archive/a0fbf98d4e3a232144df4d0d80b577c7a693b570.tar.gz"], use_category = ["build"], ), six = dict( diff --git a/tools/config_validation/BUILD b/tools/config_validation/BUILD new file mode 100644 index 000000000000..a6e8b6db72b2 --- /dev/null +++ b/tools/config_validation/BUILD @@ -0,0 +1,10 @@ +licenses(["notice"]) # Apache 2 + +load("@config_validation//:requirements.bzl", "requirement") + +py_binary( + name = "validate_yaml_fragment", + srcs = ["validate_yaml_fragment.py"], + visibility = ["//visibility:public"], + deps = [requirement("PyYAML")], +) diff --git a/tools/config_validation/requirements.txt b/tools/config_validation/requirements.txt new file mode 100644 index 000000000000..7a997b5e44bd --- /dev/null +++ b/tools/config_validation/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.3.1 diff --git a/tools/config_validation/validate_yaml_fragment.py b/tools/config_validation/validate_yaml_fragment.py new file mode 100644 index 000000000000..0cfac273b237 --- /dev/null +++ b/tools/config_validation/validate_yaml_fragment.py @@ -0,0 +1,3 @@ +import yaml + +print('YAML version is %s' % yaml.__version__) From f806aa8f1d9574b4d780a7ca113aaa260b2046f0 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 7 May 2020 18:05:10 -0400 Subject: [PATCH 02/10] Update WORKSPACE.filter.example as well. Signed-off-by: Harvey Tuch --- ci/WORKSPACE.filter.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index db20a3146ac9..5f41e02a5850 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -17,6 +17,10 @@ load("@envoy//bazel:repositories.bzl", "envoy_dependencies") envoy_dependencies() +load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra") + +envoy_dependencies_extra() + load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports") envoy_dependency_imports() From f053e99484f78c25f4843529cd1823e9da6e74f7 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 7 May 2020 18:13:31 -0400 Subject: [PATCH 03/10] Document in bazel/EXTERNAL_DEPS.md. Signed-off-by: Harvey Tuch --- bazel/EXTERNAL_DEPS.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md index 7793129376aa..34300866d54d 100644 --- a/bazel/EXTERNAL_DEPS.md +++ b/bazel/EXTERNAL_DEPS.md @@ -6,7 +6,9 @@ values can change when Github change their tar/gzip libraries breaking builds. Maintainer provided tarballs are more stable and the maintainer can provide the SHA256. -# Adding external dependencies to Envoy (native Bazel) +# Adding external dependencies to Envoy (C++) + +## Native Bazel This is the preferred style of adding dependencies that use Bazel for their build process. @@ -17,7 +19,7 @@ build process. `external_deps` attribute. 3. `bazel test //test/...` -# Adding external dependencies to Envoy (external CMake) +## External CMake (preferred) This is the preferred style of adding dependencies that use CMake for their build system. @@ -29,7 +31,8 @@ This is the preferred style of adding dependencies that use CMake for their buil `external_deps` attribute. 4. `bazel test //test/...` -# Adding external dependencies to Envoy (genrule repository) + +## genrule repository This is the newer style of adding dependencies with no upstream Bazel configs. It wraps the dependency's native build tooling in a Bazel-aware shell script, @@ -54,6 +57,24 @@ Dependencies between external libraries can use the standard Bazel dependency resolution logic, using the `$(location)` shell extension to resolve paths to binaries, libraries, headers, etc. +# Adding external dependencies to Envoy (Python) + +Python dependencies should be added via `pip3` and `rules_python`. The process +is: + +1. Define a `pip3_import()` pointing at your target `requirements.txt` in + [`bazel/repositories_extra.bzl`](repositories_extra.bzl) + +2. Add a `pip_install()` invocation in + [`bazel/dependency_imports.bzl`](dependency_imports.bzl). + +3. Add a `requirements(" Date: Thu, 7 May 2020 18:17:10 -0400 Subject: [PATCH 04/10] Fix tools path. Signed-off-by: Harvey Tuch --- bazel/EXTERNAL_DEPS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/EXTERNAL_DEPS.md b/bazel/EXTERNAL_DEPS.md index 34300866d54d..7eebe1c3ec2b 100644 --- a/bazel/EXTERNAL_DEPS.md +++ b/bazel/EXTERNAL_DEPS.md @@ -71,7 +71,7 @@ is: 3. Add a `requirements(" Date: Thu, 7 May 2020 19:00:14 -0400 Subject: [PATCH 05/10] Kick CI Signed-off-by: Harvey Tuch From 82bbc6ada009aec8ae58bafb4346484fd6f1f3ef Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 7 May 2020 19:46:39 -0400 Subject: [PATCH 06/10] Kick CI Signed-off-by: Harvey Tuch From af77a09958aac7972d582323a42cd9ee6e6ea65f Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Fri, 8 May 2020 06:12:25 -0400 Subject: [PATCH 07/10] Qualify repo in ci/WORKSPACE.filter.example. Signed-off-by: Harvey Tuch --- ci/WORKSPACE.filter.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index 5f41e02a5850..0159ddfd7821 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -17,7 +17,7 @@ load("@envoy//bazel:repositories.bzl", "envoy_dependencies") envoy_dependencies() -load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra") +load("@envoy//bazel:repositories_extra.bzl", "envoy_dependencies_extra") envoy_dependencies_extra() From 2ff5042b90b92d9de4d761ebd83c3353817f000b Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Fri, 8 May 2020 08:34:19 -0400 Subject: [PATCH 08/10] More qualification. Signed-off-by: Harvey Tuch --- bazel/repositories_extra.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index 277b6b6736c8..fe0e9adb6a29 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -8,7 +8,7 @@ def _python_deps(): pip3_import( name = "config_validation", - requirements = "//tools/config_validation:requirements.txt", + requirements = "@envoy//tools/config_validation:requirements.txt", ) # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies(). From 4a1c3e011a0ee7e0a59a1194393acc7fd7194124 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Mon, 11 May 2020 13:17:34 -0400 Subject: [PATCH 09/10] Update Windows CI build hash. Signed-off-by: Harvey Tuch --- ci/run_envoy_docker_windows.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_envoy_docker_windows.sh b/ci/run_envoy_docker_windows.sh index 023234e21b71..9594f3f2148a 100644 --- a/ci/run_envoy_docker_windows.sh +++ b/ci/run_envoy_docker_windows.sh @@ -4,7 +4,7 @@ set -e # TODO(sunjayBhatia, wrowe): update this note once we have an RBE toolchain generated for Windows # NOTE: Update this from the latest pushed image here: https://hub.docker.com/r/envoyproxy/envoy-build-windows2019/tags -ENVOY_BUILD_SHA="3cbc11e373dc4e3a523b9273ed010c5e0f197874" +ENVOY_BUILD_SHA="596a50d2aa5bd62d233273b4abce967d9dda4bab" [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019" # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker From 0c8814bb5521d944a89f011ed6b68653c13fe8b2 Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Tue, 12 May 2020 12:23:01 -0400 Subject: [PATCH 10/10] Bump Windows Docker SHA again to pickup pip wheel fixes. Signed-off-by: Harvey Tuch --- ci/run_envoy_docker_windows.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_envoy_docker_windows.sh b/ci/run_envoy_docker_windows.sh index 9594f3f2148a..6c2cb9e4015f 100644 --- a/ci/run_envoy_docker_windows.sh +++ b/ci/run_envoy_docker_windows.sh @@ -4,7 +4,7 @@ set -e # TODO(sunjayBhatia, wrowe): update this note once we have an RBE toolchain generated for Windows # NOTE: Update this from the latest pushed image here: https://hub.docker.com/r/envoyproxy/envoy-build-windows2019/tags -ENVOY_BUILD_SHA="596a50d2aa5bd62d233273b4abce967d9dda4bab" +ENVOY_BUILD_SHA="9b7dc527351b9888805377a05e5975d6ef8d6ae1" [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019" # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker