diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 3dca064039fb..7d4923d6cffb 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -22,6 +22,8 @@ local math = math local type = type local core = require("apisix.core") local http = require("resty.http") +local patch = require("apisix.patch") + local function list_query(informer) local arguments = { @@ -45,15 +47,18 @@ end local function list(httpc, apiserver, informer) + local headers = { + ["Host"] = apiserver.host .. ":" .. apiserver.port, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + if apiserver.token and apiserver.token ~= "" then + headers["Authorization"] = "Bearer " .. apiserver.token + end local response, err = httpc:request({ path = informer.path, query = list_query(informer), - headers = { - ["Host"] = apiserver.host .. ":" .. apiserver.port, - ["Authorization"] = "Bearer " .. apiserver.token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } + headers = headers }) core.log.info("--raw=", informer.path, "?", list_query(informer)) @@ -204,15 +209,18 @@ local function watch(httpc, apiserver, informer) local http_seconds = watch_seconds + 120 httpc:set_timeouts(2000, 3000, http_seconds * 1000) + local headers = { + ["Host"] = apiserver.host .. ":" .. apiserver.port, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + if apiserver.token and apiserver.token ~= "" then + headers["Authorization"] = "Bearer " .. apiserver.token + end local response, err = httpc:request({ path = informer.path, query = watch_query(informer), - headers = { - ["Host"] = apiserver.host .. ":" .. apiserver.port, - ["Authorization"] = "Bearer " .. apiserver.token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } + headers = headers }) core.log.info("--raw=", informer.path, "?", watch_query(informer)) @@ -269,12 +277,22 @@ local function list_watch(informer, apiserver) informer.fetch_state = "connecting" core.log.info("begin to connect ", apiserver.host, ":", apiserver.port) - ok, message = httpc:connect({ + local opt = { scheme = apiserver.schema, host = apiserver.host, port = apiserver.port, - ssl_verify = false - }) + ssl_verify = apiserver.ssl_verify, + } + if apiserver.schema == "https" and + apiserver.cert and apiserver.cert ~= "" and + apiserver.key and apiserver.key ~= "" then + opt.ssl_cert_path = apiserver.cert + opt.ssl_key_path = apiserver.key + opt.ssl_server_name = apiserver.host + -- replace tcp socket of http client to support mtls + httpc.sock = patch.lua_tcp_socket() + end + ok, message = httpc:connect(opt) if not ok then informer.fetch_state = "connect failed" diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index d7258a55642b..1191a9d1c232 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -270,15 +270,33 @@ local function get_apiserver(conf) if err then return nil, err end + elseif conf.client.cert_file and conf.client.key_file then + apiserver.cert, err = read_env(conf.client.cert_file) + if err then + return nil, err + end + apiserver.key, err = read_env(conf.client.key_file) + if err then + return nil, err + end else - return nil, "one of [client.token,client.token_file] should be set but none" + return nil, "one of [client.token,client.token_file,(client.cert_file,client.key_file)] ".. + "should be set but none" + end + + apiserver.ssl_verify = false + if conf.client.ssl_verify then + apiserver.ssl_verify = conf.client.ssl_verify end -- remove possible extra whitespace apiserver.token = apiserver.token:gsub("%s+", "") - if apiserver.schema == "https" and apiserver.token == "" then - return nil, "apiserver.token should set to non-empty string when service.schema is https" + if apiserver.schema == "https" then + if apiserver.token == "" and (apiserver.cert == "" or apiserver.key == "") then + return nil, "apiserver.token or (apiserver.cert and apiserver.key) ".. + "should set to non-empty string when service.schema is https" + end end return apiserver diff --git a/apisix/discovery/kubernetes/schema.lua b/apisix/discovery/kubernetes/schema.lua index 170608f553b9..52926ebe5b64 100644 --- a/apisix/discovery/kubernetes/schema.lua +++ b/apisix/discovery/kubernetes/schema.lua @@ -41,13 +41,17 @@ local token_schema = { oneOf = token_patterns, } -local token_file_schema = { +local file_schema = { type = "string", pattern = [[^[^\:*?"<>|]*$]], minLength = 1, maxLength = 500, } +local ssl_verify_schema = { + type = "boolean", +} + local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]] local namespace_regex_pattern = [[^[\x21-\x7e]*$]] @@ -135,7 +139,10 @@ return { type = "object", properties = { token = token_schema, - token_file = token_file_schema, + token_file = file_schema, + cert_file = file_schema, + key_file = file_schema, + ssl_verify = ssl_verify_schema, }, default = { token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" @@ -145,6 +152,7 @@ return { anyOf = { { required = { "token" } }, { required = { "token_file" } }, + { required = { "cert_file", "key_file" } }, } } }, @@ -191,11 +199,15 @@ return { type = "object", properties = { token = token_schema, - token_file = token_file_schema, + token_file = file_schema, + cert_file = file_schema, + key_file = file_schema, + ssl_verify = ssl_verify_schema, }, oneOf = { { required = { "token" } }, { required = { "token_file" } }, + { required = { "cert_file", "key_file" } }, }, }, namespace_selector = namespace_selector_schema, diff --git a/apisix/patch.lua b/apisix/patch.lua index 2b191b2a83e1..5f1e4c56a2d0 100644 --- a/apisix/patch.lua +++ b/apisix/patch.lua @@ -380,5 +380,8 @@ function _M.patch() end end +function _M.lua_tcp_socket() + return luasocket_tcp() +end return _M diff --git a/ci/kubernetes-ci.sh b/ci/kubernetes-ci.sh index c40b8c78c897..6341a66fc2c3 100755 --- a/ci/kubernetes-ci.sh +++ b/ci/kubernetes-ci.sh @@ -21,6 +21,7 @@ run_case() { export_or_prefix export PERL5LIB=.:$PERL5LIB + export KUBERNETES_APISERVER_ADDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o=jsonpath="{.items[0].metadata.annotations.kubeadm\.kubernetes\.io/kube-apiserver\.advertise-address\.endpoint}" | awk -F: '{print $1}') prove -Itest-nginx/lib -I./ -r t/kubernetes | tee test-result rerun_flaky_tests test-result } diff --git a/t/certs/k8s_mtls_csr.conf b/t/certs/k8s_mtls_csr.conf new file mode 100644 index 000000000000..ecf4842a5836 --- /dev/null +++ b/t/certs/k8s_mtls_csr.conf @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[req] +default_bits = 2048 +default_md = sha256 +distinguished_name = dn +prompt = no + +[dn] +CN = system:serviceaccount:default:apisix-test +O = system:serviceaccounts + +[v3_ext] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = CA:TRUE +keyUsage = keyEncipherment,dataEncipherment +extendedKeyUsage = clientAuth + +## openssl genrsa -out k8s_mtls.key 4096 +## openssl req -new -key k8s_mtls.key -config k8s_mtls_csr.conf -out k8s_mtls.csr -nodes + diff --git a/t/certs/k8s_mtls_csr.yaml b/t/certs/k8s_mtls_csr.yaml new file mode 100644 index 000000000000..ec8a3e438743 --- /dev/null +++ b/t/certs/k8s_mtls_csr.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: certificates.k8s.io/v1 +kind: CertificateSigningRequest +metadata: + name: k8s-mtls-csr +spec: + groups: + - system:authenticated + request: ${BASE64_CSR} + signerName: kubernetes.io/kube-apiserver-client + usages: + - digital signature + - key encipherment + - client auth diff --git a/t/kubernetes/discovery/kubernetes.t b/t/kubernetes/discovery/kubernetes.t index 24bf6f426a01..365e377b7119 100644 --- a/t/kubernetes/discovery/kubernetes.t +++ b/t/kubernetes/discovery/kubernetes.t @@ -22,8 +22,16 @@ no_root_location(); no_shuffle(); workers(4); +system('grep client-cert ~/.kube/config |cut -d" " -f 6 | base64 -d > ./t/certs/k8s_mtls.pem'); +system('grep client-key-data ~/.kube/config |cut -d" " -f 6 | base64 -d > ./t/certs/k8s_mtls.key'); +system('grep certificate-authority-data ~/.kube/config |cut -d" " -f 6 | base64 -d > ./t/certs/k8s_mtls_ca.pem'); + +our $apiserver_addr = $ENV{'KUBERNETES_APISERVER_ADDR'} || "127.0.0.1"; our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; our $token_value = eval {`cat $token_file 2>/dev/null`}; +our $cert_file = "./t/certs/k8s_mtls.pem"; +our $key_file = "./t/certs/k8s_mtls.key"; +our $ca_cert_file = "./t/certs/k8s_mtls_ca.pem"; add_block_preprocessor(sub { my ($block) = @_; @@ -37,9 +45,12 @@ _EOC_ my $main_config = $block->main_config // <<_EOC_; env MyPort=6443; -env KUBERNETES_SERVICE_HOST=127.0.0.1; +env KUBERNETES_SERVICE_HOST=$::apiserver_addr; env KUBERNETES_SERVICE_PORT=6443; env KUBERNETES_CLIENT_TOKEN=$::token_value; +env KUBERNETES_CLIENT_CERT=$::cert_file; +env KUBERNETES_CLIENT_KEY=$::key_file; +env KUBERNETES_CERTIFICATE_AUTHORITY=$::ca_cert_file; _EOC_ $block->set_value("main_config", $main_config); @@ -304,3 +315,150 @@ GET /compare Content-type: application/json --- response_body true + + + +=== TEST 6: default value with minimal client tls configuration +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + kubernetes: + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} +--- request +GET /compare +{ + "service": { + "schema": "https", + "host": "${KUBERNETES_SERVICE_HOST}", + "port": "${KUBERNETES_SERVICE_PORT}" + }, + "client": { + "cert_file": "${KUBERNETES_CLIENT_CERT}", + "key_file": "${KUBERNETES_CLIENT_KEY}" + }, + "shared_size": "1m", + "default_weight": 50 +} +--- more_headers +Content-type: application/json +--- response_body +true + + + +=== TEST 7: client tls configuration with ca cert +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_trusted_certificate: t/certs/k8s_mtls_ca.pem + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: true +--- request +GET /compare +{ + "service": { + "schema": "https", + "host": "${KUBERNETES_SERVICE_HOST}", + "port": "${KUBERNETES_SERVICE_PORT}" + }, + "client": { + "cert_file": "${KUBERNETES_CLIENT_CERT}", + "key_file": "${KUBERNETES_CLIENT_KEY}", + "ssl_verify": true + }, + "shared_size": "1m", + "default_weight": 50 +} +--- more_headers +Content-type: application/json +--- response_body +true + + + +=== TEST 8: multi cluster mode client tls configuration +--- http_config +lua_shared_dict kubernetes-debug 1m; +lua_shared_dict kubernetes-release 1m; +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + kubernetes: + - id: "debug" + service: + schema: "http" + host: "1.cluster.com" + port: "6445" + client: + token: ${KUBERNETES_CLIENT_TOKEN} + - id: "release" + service: + schema: "https" + host: "2.cluster.com" + port: "${MyPort}" + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: false + default_weight: 33 + shared_size: "2m" +--- request +GET /compare +[ + { + "id": "debug", + "service": { + "schema": "http", + "host": "1.cluster.com", + "port": "6445" + }, + "client": { + "token": "${KUBERNETES_CLIENT_TOKEN}" + }, + "default_weight": 50, + "shared_size": "1m" + }, + { + "id": "release", + "service": { + "schema": "https", + "host": "2.cluster.com", + "port": "${MyPort}" + }, + "client": { + "cert_file": "${KUBERNETES_CLIENT_CERT}", + "key_file": "${KUBERNETES_CLIENT_KEY}", + "ssl_verify": false + }, + "default_weight": 33, + "shared_size": "2m" + } +] +--- more_headers +Content-type: application/json +--- response_body +true diff --git a/t/kubernetes/discovery/kubernetes_mtls.t b/t/kubernetes/discovery/kubernetes_mtls.t new file mode 100644 index 000000000000..df7212c6a878 --- /dev/null +++ b/t/kubernetes/discovery/kubernetes_mtls.t @@ -0,0 +1,456 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BEGIN { + our $apiserver_addr = $ENV{'KUBERNETES_APISERVER_ADDR'}; + our $cert_file = "./t/certs/k8s_mtls.pem"; + our $key_file = "./t/certs/k8s_mtls.key"; + our $ca_cert_file = "./t/certs/k8s_mtls_ca.pem"; + our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; + our $token_value = eval {`cat $token_file 2>/dev/null`}; + + our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_trusted_certificate: $::ca_cert_file + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: $::apiserver_addr + port: "6443" + client: + cert_file: $::cert_file + key_file: $::key_file + ssl_verify: true + - id: second + service: + schema: "http" + host: "127.0.0.1" + port: "6445" + client: + token_file: "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token" +_EOC_ + + our $scale_ns_c = <<_EOC_; +[ + { + "op": "replace_subsets", + "name": "ep", + "namespace": "ns-c", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + } + ] + } +] +_EOC_ + +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('warn'); +no_root_location(); +no_shuffle(); +workers(4); + +add_block_preprocessor(sub { + my ($block) = @_; + + my $apisix_yaml = $block->apisix_yaml // <<_EOC_; +routes: [] +#END +_EOC_ + + $block->set_value("apisix_yaml", $apisix_yaml); + + my $main_config = $block->main_config // <<_EOC_; +env KUBERNETES_SERVICE_HOST=$::apiserver_addr; +env KUBERNETES_SERVICE_PORT=6443; +env KUBERNETES_CLIENT_CERT=$::cert_file; +env KUBERNETES_CLIENT_KEY=$::key_file; +env KUBERNETES_CERTIFICATE_AUTHORITY=$::ca_cert_file; +env KUBERNETES_CLIENT_TOKEN=$::token_value; +_EOC_ + + $block->set_value("main_config", $main_config); + + my $config = $block->config // <<_EOC_; + location /queries { + content_by_lua_block { + local core = require("apisix.core") + local d = require("apisix.discovery.kubernetes") + + ngx.sleep(1) + + ngx.req.read_body() + local request_body = ngx.req.get_body_data() + local queries = core.json.decode(request_body) + local response_body = "{" + for _,query in ipairs(queries) do + local nodes = d.nodes(query) + if nodes==nil or #nodes==0 then + response_body=response_body.." "..0 + else + response_body=response_body.." "..#nodes + end + end + ngx.say(response_body.." }") + } + } + + location /operators { + content_by_lua_block { + local http = require("resty.http") + local core = require("apisix.core") + local ipairs = ipairs + + ngx.req.read_body() + local request_body = ngx.req.get_body_data() + local operators = core.json.decode(request_body) + + core.log.info("get body ", request_body) + core.log.info("get operators ", #operators) + for _, op in ipairs(operators) do + local method, path, body + local headers = { + ["Host"] = "127.0.0.1:6445" + } + + if op.op == "replace_subsets" then + method = "PATCH" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name + if #op.subsets == 0 then + body = '[{"path":"/subsets","op":"replace","value":[]}]' + else + local t = { { op = "replace", path = "/subsets", value = op.subsets } } + body = core.json.encode(t, true) + end + headers["Content-Type"] = "application/json-patch+json" + end + + if op.op == "replace_labels" then + method = "PATCH" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name + local t = { { op = "replace", path = "/metadata/labels", value = op.labels } } + body = core.json.encode(t, true) + headers["Content-Type"] = "application/json-patch+json" + end + + local httpc = http.new() + core.log.info("begin to connect ", "127.0.0.1:6445") + local ok, message = httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = 6445, + }) + if not ok then + core.log.error("connect 127.0.0.1:6445 failed, message : ", message) + ngx.say("FAILED") + end + local res, err = httpc:request({ + method = method, + path = path, + headers = headers, + body = body, + }) + if err ~= nil then + core.log.err("operator k8s cluster error: ", err) + return 500 + end + if res.status ~= 200 and res.status ~= 201 and res.status ~= 409 then + return res.status + end + end + ngx.say("DONE") + } + } + +_EOC_ + + $block->set_value("config", $config); + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create namespace and endpoints +--- yaml_config eval: $::yaml_config +--- request +POST /operators +[ + { + "op": "replace_subsets", + "namespace": "ns-a", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "name": "p2", + "port": 5002 + } + ] + } + ] + }, + { + "op": "create_namespace", + "name": "ns-b" + }, + { + "op": "replace_subsets", + "namespace": "ns-b", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "name": "p2", + "port": 5002 + } + ] + } + ] + }, + { + "op": "create_namespace", + "name": "ns-c" + }, + { + "op": "replace_subsets", + "namespace": "ns-c", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "port": 5002 + } + ] + } + ] + } +] +--- more_headers +Content-type: application/json + + + +=== TEST 2: use default parameters +--- yaml_config eval: $::yaml_config +--- request +GET /queries +[ + "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002" +] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } + + + +=== TEST 3: use client tls without ca certificate +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: ${KUBERNETES_SERVICE_HOST} + port: ${KUBERNETES_SERVICE_PORT} + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: false + +--- request +GET /queries +[ + "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002" +] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } + + + +=== TEST 4: scale endpoints +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_trusted_certificate: ${KUBERNETES_CERTIFICATE_AUTHORITY} + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: ${KUBERNETES_SERVICE_HOST} + port: ${KUBERNETES_SERVICE_PORT} + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: true + +--- request eval +[ + +"GET /queries +[ + \"first/ns-a/ep:p1\",\"first/ns-a/ep:p2\" +]", + +"POST /operators +[{\"op\":\"replace_subsets\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"subsets\":[]}]", + +"GET /queries +[ + \"first/ns-a/ep:p1\",\"first/ns-a/ep:p2\" +]", + +"GET /queries +[ + \"first/ns-c/ep:5001\",\"first/ns-c/ep:5002\",\"first/ns-c/ep:p1\" +]", + +"POST /operators +$::scale_ns_c", + +"GET /queries +[ + \"first/ns-c/ep:5001\",\"first/ns-c/ep:5002\",\"first/ns-c/ep:p1\" +]" + +] +--- response_body eval +[ + "{ 2 2 }\n", + "DONE\n", + "{ 0 0 }\n", + "{ 2 2 0 }\n", + "DONE\n", + "{ 0 0 1 }\n", +]