From d0d0d76b96c7775c1334ea81f1add59909a1995a Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 22 Nov 2022 15:05:59 +0800 Subject: [PATCH 01/27] feat(discovery): add consul discovery --- apisix/discovery/consul/init.lua | 428 +++++++++++++++++++++++++++++++ 1 file changed, 428 insertions(+) create mode 100644 apisix/discovery/consul/init.lua diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua new file mode 100644 index 000000000000..d9124067b0d8 --- /dev/null +++ b/apisix/discovery/consul/init.lua @@ -0,0 +1,428 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local require = require +local local_conf = require("apisix.core.config_local").local_conf() +local core = require("apisix.core") +local core_sleep = require("apisix.core.utils").sleep +local resty_consul = require('resty.consul') +local http = require('resty.http') +local util = require("apisix.cli.util") +local ipairs = ipairs +local error = error +local ngx = ngx +local unpack = unpack +local tonumber = tonumber +local pairs = pairs +local ngx_timer_at = ngx.timer.at +local ngx_timer_every = ngx.timer.every +local log = core.log +local json_delay_encode = core.json.delay_encode + +local all_services = core.table.new(0, 5) +local default_service +local default_weight +local skip_service_map = core.table.new(0, 1) +local dump_params + +local events +local events_list +local consul_services + +local _M = { + version = 0.1, +} + + +local function discovery_consul_callback(data, event, source, pid) + all_services = data + log.notice("update local variable all_services, event is: ", event, + "source: ", source, "server pid:", pid, + ", all services: ", core.json.encode(all_services, true)) +end + + +function _M.all_nodes() + return all_services +end + + +function _M.nodes(service_name) + if not all_services then + log.error("all_services is nil, failed to fetch nodes for : ", service_name) + return + end + + local resp_list = all_services[service_name] + + if not resp_list then + log.error("fetch nodes failed by ", service_name, ", return default service") + return default_service and {default_service} + end + + log.info("process id: ", ngx.worker.id(), ", all_services[", service_name, "] = ", + json_delay_encode(resp_list, true)) + + return resp_list +end + + +local function parse_instance(node) + local service_name, host, port = node.Service, node.Address, node.Port + -- if exist, skip special service name + if service_name and skip_service_map[service_name] then + return false + end + + -- if host is empty, get host and port from TaggedAddresses.wan + if not host then + local tagged_addresses = node.TaggedAddresses + if tagged_addresses then + host = tagged_addresses.wan.address + port = tagged_addresses.wan.port + end + end + + return true, host, tonumber(port), "", service_name +end + + +local function update_all_services(server_name_prefix, data) + local sn + local up_services = core.table.new(0, #data) + local weight = default_weight + for _, node in pairs(data) do + local succ, ip, port, metadata, server_name = parse_instance(node) + if succ then + sn = server_name + local nodes = up_services[sn] + if not nodes then + nodes = core.table.new(1, 0) + up_services[sn] = nodes + end + core.table.insert(nodes, { + host = ip, + port = port, + weight = metadata and metadata.weight or weight, + }) + end + end + + -- clean old unused data + local old_services = consul_services[server_name_prefix] or {} + for k, _ in pairs(old_services) do + all_services[k] = nil + end + core.table.clear(old_services) + + for k, v in pairs(up_services) do + all_services[k] = v + end + consul_services[server_name_prefix] = up_services + + log.info("update all services: ", core.json.encode(all_services)) +end + + +local function read_dump_services() + local data, err = util.read_file(dump_params.path) + if not data then + log.error("read dump file get error: ", err) + return + end + + log.info("read dump file: ", data) + data = util.trim(data) + if #data == 0 then + log.error("dump file is empty") + return + end + + local entity, err = core.json.decode(data) + if not entity then + log.error("decoded dump data got error: ", err, ", file content: ", data) + return + end + + if not entity.services or not entity.last_update then + log.warn("decoded dump data miss fields, file content: ", data) + return + end + + local now_time = ngx.time() + log.info("dump file last_update: ", entity.last_update, ", dump_params.expire: ", + dump_params.expire, ", now_time: ", now_time) + if dump_params.expire ~= 0 and (entity.last_update + dump_params.expire) < now_time then + log.warn("dump file: ", dump_params.path, " had expired, ignored it") + return + end + + all_services = entity.services + log.info("load dump file into memory success") +end + + +local function write_dump_services() + local entity = { + services = all_services, + last_update = ngx.time(), + expire = dump_params.expire, -- later need handle it + } + local data = core.json.encode(entity) + local succ, err = util.write_file(dump_params.path, data) + if not succ then + log.error("write dump into file got error: ", err) + end +end + + +local function show_dump_file() + if not dump_params then + return 503, "dump params is nil" + end + + local data, err = util.read_file(dump_params.path) + if not data then + return 503, err + end + + return 200, data +end + + +function _M.connect(premature, consul_server, retry_delay) + if premature then + return + end + + local consul_client = resty_consul:new({ + host = consul_server.host, + port = consul_server.port, + connect_timeout = consul_server.connect_timeout, + read_timeout = consul_server.read_timeout, + default_args = consul_server.default_args, + }) + + log.info("consul_server: ", json_delay_encode(consul_server, true)) + local watch_result, watch_err = consul_client:get(consul_server.consul_watch_sub_url) + local watch_error_info = (watch_err ~= nil and watch_err) + or ((watch_result ~= nil and watch_result.status ~= 200) + and watch_result.status) + if watch_error_info then + log.error("connect consul: ", consul_server.consul_server_url, " by sub url: ", consul_server.consul_sub_url, + ", got watch result: ", json_delay_encode(watch_result, true), ", with error: ", watch_error_info) + + if not retry_delay then + retry_delay = 1 + else + retry_delay = retry_delay * 4 + end + + log.warn("retry connecting consul after ", retry_delay, " seconds") + core_sleep(retry_delay) + + goto ERR + end + + log.info("connect consul: ", consul_server.consul_server_url, + ", watch_result status: ", watch_result.status, + ", watch_result.headers.index: ", watch_result.headers['X-Consul-Index'], + ", consul_server.index: ", consul_server.index, + ", consul_server: ", json_delay_encode(consul_server)) + + -- if current index different last index then update service + if consul_server.index ~= watch_result.headers['X-Consul-Index'] then + + -- fetch all services info + local result, err = consul_client:get(consul_server.consul_sub_url) + + local error_info = (err ~= nil and err) or ((result ~= nil and result.status ~= 200) and result.status) + + if error_info then + log.error("connect consul: ", consul_server.consul_server_url, + " by sub url: ", consul_server.consul_sub_url, + ", got result: ", json_delay_encode(result, true), + ", with error: ", error_info) + + if not retry_delay then + retry_delay = 1 + else + retry_delay = retry_delay * 4 + end + + log.warn("retry connecting consul after ", retry_delay, " seconds") + core_sleep(retry_delay) + + goto ERR + end + + consul_server.index = watch_result.headers['X-Consul-Index'] + -- only long connect type use index + if consul_server.keepalive then + consul_server.default_args.index = watch_result.headers['X-Consul-Index'] + end + -- decode body, decode json, update service, error handling + if result.body and next(result.body) then + log.notice("server_name: ", consul_server.consul_server_url, + ", header: ", core.json.encode(result.headers, true), + ", body: ", core.json.encode(result.body, true)) + update_all_services(consul_server.consul_server_url, result.body) + --update events + local ok, err = events.post(events_list._source, events_list.updating, all_services) + if not ok then + log.error("post_event failure with ", events_list._source, + ", update all services error: ", err) + end + + if dump_params then + ngx_timer_at(0, write_dump_services) + end + end + end + + :: ERR :: + local keepalive = consul_server.keepalive + if keepalive then + local ok, err = ngx_timer_at(0, _M.connect, consul_server, retry_delay) + if not ok then + log.error("create ngx_timer_at got error: ", err) + return + end + end +end + + +local function format_consul_params(consul_conf) + local consul_server_list = core.table.new(0, #consul_conf.servers) + local args + + if consul_conf.keepalive == false then + args = { + recurse = true, + } + elseif consul_conf.keepalive then + args = { + recurse = true, + wait = consul_conf.timeout.wait, --blocked wait!=0; unblocked by wait=0 + index = 0, + } + end + + for _, v in pairs(consul_conf.servers) do + local scheme, host, port, path = unpack(http.parse_uri(nil, v)) + if scheme ~= "http" then + return nil, "only support consul http schema address, eg: http://address:port" + elseif path ~= "/" or core.string.has_suffix(v, '/') then + return nil, "invalid consul server address, the valid format: http://address:port" + end + + core.table.insert(consul_server_list, { + host = host, + port = port, + connect_timeout = consul_conf.timeout.connect, + read_timeout = consul_conf.timeout.read, + consul_sub_url = "/agent/services", + consul_watch_sub_url = "/catalog/services", + consul_server_url = v .. "/v1", + weight = consul_conf.weight, + keepalive = consul_conf.keepalive, + default_args = args, + index = 0, + fetch_interval = consul_conf.fetch_interval -- fetch interval to next connect consul + }) + end + + return consul_server_list +end + + +function _M.init_worker() + local consul_conf = local_conf.discovery.consul + + if consul_conf.dump then + local dump = consul_conf.dump + dump_params = dump + + if dump.load_on_init then + read_dump_services() + end + end + + events = require("resty.worker.events") + events_list = events.event_list( + "discovery_consul_update_all_services", + "updating" + ) + + if 0 ~= ngx.worker.id() then + events.register(discovery_consul_callback, events_list._source, events_list.updating) + return + end + + log.notice("consul_conf: ", core.json.encode(consul_conf)) + default_weight = consul_conf.weight + -- set default service, used when the server node cannot be found + if consul_conf.default_service then + default_service = consul_conf.default_service + default_service.weight = default_weight + end + if consul_conf.skip_services then + skip_service_map = core.table.new(0, #consul_conf.skip_services) + for _, v in ipairs(consul_conf.skip_services) do + skip_service_map[v] = true + end + end + + local consul_servers_list, err = format_consul_params(consul_conf) + if err then + error(err) + return + end + log.info("consul_server_list: ", core.json.encode(consul_servers_list)) + + consul_services = core.table.new(0, 1) + -- success or failure + for _, server in ipairs(consul_servers_list) do + local ok, err = ngx_timer_at(0, _M.connect, server) + if not ok then + error("create consul got error: " .. err) + return + end + + if server.keepalive == true then + ngx_timer_every(server.fetch_interval, _M.connect, server) + end + end +end + + +function _M.dump_data() + return {config = local_conf.discovery.consul, services = all_services } +end + + +function _M.control_api() + return { + { + methods = {"GET"}, + uris = {"/show_dump_file"}, + handler = show_dump_file, + } + } +end + + +return _M From 9e983a3e62d2786421b58c0e2f7ad37575c6173a Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 22 Nov 2022 15:06:56 +0800 Subject: [PATCH 02/27] feat(discovery): add consul discovery --- apisix/discovery/consul/schema.lua | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 apisix/discovery/consul/schema.lua diff --git a/apisix/discovery/consul/schema.lua b/apisix/discovery/consul/schema.lua new file mode 100644 index 000000000000..3e998b015ce1 --- /dev/null +++ b/apisix/discovery/consul/schema.lua @@ -0,0 +1,86 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +return { + type = "object", + properties = { + servers = { + type = "array", + minItems = 1, + items = { + type = "string", + } + }, + fetch_interval = {type = "integer", minimum = 1, default = 3}, + keepalive = { + type = "boolean", + default = true + }, + weight = {type = "integer", minimum = 1, default = 1}, + timeout = { + type = "object", + properties = { + connect = {type = "integer", minimum = 1, default = 2000}, + read = {type = "integer", minimum = 1, default = 2000}, + wait = {type = "integer", minimum = 1, default = 60} + }, + default = { + connect = 2000, + read = 2000, + wait = 60, + } + }, + skip_services = { + type = "array", + minItems = 1, + items = { + type = "string", + } + }, + dump = { + type = "object", + properties = { + path = {type = "string", minLength = 1}, + load_on_init = {type = "boolean", default = true}, + expire = {type = "integer", default = 0}, + }, + required = {"path"}, + }, + default_service = { + type = "object", + properties = { + host = {type = "string"}, + port = {type = "integer"}, + metadata = { + type = "object", + properties = { + fail_timeout = {type = "integer", default = 1}, + weight = {type = "integer", default = 1}, + max_fails = {type = "integer", default = 1} + }, + default = { + fail_timeout = 1, + weight = 1, + max_fails = 1 + } + } + } + } + }, + + required = {"servers"} +} + From b052655e06dde87fba12f21487684fd30b6653c1 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 22 Nov 2022 15:08:28 +0800 Subject: [PATCH 03/27] feat(discovery): add consul discovery --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 49468dc57e41..41d070ac1c49 100644 --- a/Makefile +++ b/Makefile @@ -283,8 +283,9 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery $(ENV_INSTALL) apisix/discovery/*.lua $(ENV_INST_LUADIR)/apisix/discovery/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul_kv,dns,eureka,nacos,kubernetes,tars} + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul_kv,consul,dns,eureka,nacos,kubernetes,tars} $(ENV_INSTALL) apisix/discovery/consul_kv/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul_kv + $(ENV_INSTALL) apisix/discovery/consul/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns $(ENV_INSTALL) apisix/discovery/eureka/*.lua $(ENV_INST_LUADIR)/apisix/discovery/eureka $(ENV_INSTALL) apisix/discovery/kubernetes/*.lua $(ENV_INST_LUADIR)/apisix/discovery/kubernetes From b80432ece66955d34069769c2ab25dd481adabd2 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 22 Nov 2022 15:10:04 +0800 Subject: [PATCH 04/27] feat(discovery): add consul discovery --- conf/config-default.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 6e714b577346..c1ccc6bfd77f 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -309,6 +309,29 @@ nginx_config: # config for render the template to generate n # dump: # if you need, when registered nodes updated can dump into file # path: "logs/consul_kv.dump" # expire: 2592000 # unit sec, here is 30 day +# consul: +# servers: +# - "http://127.0.0.1:8500" +# - "http://127.0.0.1:8600" +# skip_services: # if you need to skip special keys +# - "service_a" +# timeout: +# connect: 2000 # default 2000 ms +# read: 2000 # default 2000 ms +# wait: 60 # default 60 sec +# weight: 1 # default 1 +# fetch_interval: 3 # default 3 sec, only take effect for keepalive: false way +# keepalive: true # default true, use the long pull way to query consul servers +# default_service: # you can define default server when missing hit +# host: "127.0.0.1" +# port: 20999 +# metadata: +# fail_timeout: 1 # default 1 ms +# weight: 1 # default 1 +# max_fails: 1 # default 1 +# dump: # if you need, when registered nodes updated can dump into file +# path: "logs/consul.dump" +# expire: 2592000 # unit sec, here is 30 day # kubernetes: # ### kubernetes service discovery both support single-cluster and multi-cluster mode # ### applicable to the case where the service is distributed in a single or multiple kubernetes clusters. From e8258f199ec8282b1b670ff97bb295c1f402face Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 22 Nov 2022 15:11:24 +0800 Subject: [PATCH 05/27] feat(discovery): add consul discovery --- t/discovery/consul.t | 570 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 570 insertions(+) create mode 100644 t/discovery/consul.t diff --git a/t/discovery/consul.t b/t/discovery/consul.t new file mode 100644 index 000000000000..70ee221ec52d --- /dev/null +++ b/t/discovery/consul.t @@ -0,0 +1,570 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); + + +add_block_preprocessor(sub { + my ($block) = @_; + + my $http_config = $block->http_config // <<_EOC_; + + server { + listen 20999; + + location / { + content_by_lua_block { + ngx.say("missing consul services") + } + } + } + + server { + listen 30511; + + location /hello { + content_by_lua_block { + ngx.say("server 1") + } + } + } + server { + listen 30512; + + location /hello { + content_by_lua_block { + ngx.say("server 2") + } + } + } + server { + listen 30513; + + location /hello { + content_by_lua_block { + ngx.say("server 3") + } + } + } + server { + listen 30514; + + location /hello { + content_by_lua_block { + ngx.say("server 4") + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + - "http://127.0.0.1:8600" + prefix: "upstreams" + skip_keys: + - "upstreams/unused_api/" + timeout: + connect: 1000 + read: 1000 + wait: 60 + weight: 1 + fetch_interval: 1 + keepalive: true + default_service: + host: "127.0.0.1" + port: 20999 + metadata: + fail_timeout: 1 + weight: 1 + max_fails: 1 +_EOC_ + + +run_tests(); + +__DATA__ + +=== TEST 1: prepare consul kv register nodes +--- config +location /consul1 { + rewrite ^/consul1/(.*) /v1/kv/$1 break; + proxy_pass http://127.0.0.1:8500; +} + +location /consul2 { + rewrite ^/consul2/(.*) /v1/kv/$1 break; + proxy_pass http://127.0.0.1:8600; +} +--- pipelined_requests eval +[ + "DELETE /consul1/upstreams/webpages/?recurse=true", + "DELETE /consul2/upstreams/webpages/?recurse=true", + "PUT /consul1/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /consul1/upstreams/webpages/127.0.0.1:30512\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /consul2/upstreams/webpages/127.0.0.1:30513\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /consul2/upstreams/webpages/127.0.0.1:30514\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", +] +--- response_body eval +["true", "true", "true", "true", "true", "true"] + + + +=== TEST 2: test consul server 1 +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello", +] +--- response_body_like eval +[ + qr/server [1-2]\n/, + qr/server [1-2]\n/, +] +--- no_error_log +[error, error] + + + +=== TEST 3: test consul server 2 +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8600/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- pipelined_requests eval +[ + "GET /hello", + "GET /hello" +] +--- response_body_like eval +[ + qr/server [3-4]\n/, + qr/server [3-4]\n/, +] +--- no_error_log +[error, error] + + + +=== TEST 4: test mini consul config +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + - "http://127.0.0.1:6500" +#END +--- apisix_yaml +routes: + - + uri: /hello + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- response_body_like eval +qr/server [1-2]/ +--- ignore_error_log + + + +=== TEST 5: test invalid service name +sometimes the consul key maybe deleted by mistake + +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8600/v1/kv/upstreams/deleted_keys/ + discovery_type: consul + type: roundrobin +#END +--- pipelined_requests eval +[ + "GET /hello_api", + "GET /hello_api" +] +--- response_body eval +[ + "missing consul services\n", + "missing consul services\n" +] +--- ignore_error_log + + + +=== TEST 6: test skip keys +skip some keys, return default nodes, get response: missing consul services +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8600" + prefix: "upstreams" + skip_keys: + - "upstreams/webpages/" + default_service: + host: "127.0.0.1" + port: 20999 + metadata: + fail_timeout: 1 + weight: 1 + max_fails: 1 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8600/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- response_body eval +"missing consul services\n" +--- ignore_error_log + + + +=== TEST 7: test register and unregister nodes +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +location /sleep { + content_by_lua_block { + local args = ngx.req.get_uri_args() + local sec = args.sec or "2" + ngx.sleep(tonumber(sec)) + ngx.say("ok") + } +} +--- timeout: 6 +--- request eval +[ + "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30511", + "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30512", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30513\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30514\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "GET /sleep", + + "GET /hello?random1", + "GET /hello?random2", + "GET /hello?random3", + "GET /hello?random4", + + "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30513", + "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30514", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30512\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "GET /sleep?sec=5", + + "GET /hello?random1", + "GET /hello?random2", + "GET /hello?random3", + "GET /hello?random4", + +] +--- response_body_like eval +[ + qr/true/, + qr/true/, + qr/true/, + qr/true/, + qr/ok\n/, + + qr/server [3-4]\n/, + qr/server [3-4]\n/, + qr/server [3-4]\n/, + qr/server [3-4]\n/, + + qr/true/, + qr/true/, + qr/true/, + qr/true/, + qr/ok\n/, + + qr/server [1-2]\n/, + qr/server [1-2]\n/, + qr/server [1-2]\n/, + qr/server [1-2]\n/ +] +--- ignore_error_log + + + +=== TEST 8: prepare healthy and unhealthy nodes +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "DELETE /v1/kv/upstreams/webpages/?recurse=true", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", + "PUT /v1/kv/upstreams/webpages/127.0.0.2:1988\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", +] +--- response_body eval +[ + 'true', + 'true', + 'true', +] + + + +=== TEST 9: test health checker +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uris: + - /hello + upstream_id: 1 +upstreams: + - + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin + id: 1 + checks: + active: + http_path: "/hello" + healthy: + interval: 1 + successes: 1 + unhealthy: + interval: 1 + http_failures: 1 +#END +--- config + location /thc { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + httpc:request_uri(uri, {method = "GET"}) + ngx.sleep(3) + + local code, body, res = t.test('/v1/healthcheck', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res[1].nodes, function(a, b) + return a.host < b.host + end) + ngx.say(json.encode(res)) + + local code, body, res = t.test('/v1/healthcheck/upstreams/1', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res.nodes, function(a, b) + return a.host < b.host + end) + ngx.say(json.encode(res)) + } + } +--- request +GET /thc +--- response_body +[{"healthy_nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1}],"name":"upstream#/upstreams/1","nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1},{"host":"127.0.0.2","port":1988,"priority":0,"weight":1}],"src_id":"1","src_type":"upstreams"}] +{"healthy_nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1}],"name":"upstream#/upstreams/1","nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1},{"host":"127.0.0.2","port":1988,"priority":0,"weight":1}],"src_id":"1","src_type":"upstreams"} +--- ignore_error_log + + + +=== TEST 10: clean nodes +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "DELETE /v1/kv/upstreams/webpages/?recurse=true" +] +--- response_body eval +[ + 'true' +] + + + +=== TEST 11: test consul short connect type +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + keepalive: false + fetch_interval: 3 + default_service: + host: "127.0.0.1" + port: 20999 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +location /sleep { + content_by_lua_block { + local args = ngx.req.get_uri_args() + local sec = args.sec or "2" + ngx.sleep(tonumber(sec)) + ngx.say("ok") + } +} +--- timeout: 6 +--- request eval +[ + "GET /hello", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "GET /sleep?sec=5", + "GET /hello", +] +--- response_body_like eval +[ + qr/missing consul services\n/, + qr/true/, + qr/ok\n/, + qr/server 1\n/ +] +--- ignore_error_log + + + +=== TEST 12: retry when Consul can't be reached (long connect type) +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8501" + keepalive: true + fetch_interval: 3 + default_service: + host: "127.0.0.1" + port: 20999 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8501/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- timeout: 4 +--- config +location /sleep { + content_by_lua_block { + local args = ngx.req.get_uri_args() + local sec = args.sec or "2" + ngx.sleep(tonumber(sec)) + ngx.say("ok") + } +} +--- request +GET /sleep?sec=3 +--- response_body +ok +--- grep_error_log eval +qr/retry connecting consul after \d seconds/ +--- grep_error_log_out +retry connecting consul after 1 seconds +retry connecting consul after 4 seconds From c868e4680b41a5f5aca468681b0c75ee81bc6829 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 22 Nov 2022 15:12:03 +0800 Subject: [PATCH 06/27] feat(discovery): add consul discovery --- t/discovery/consul_dump.t | 390 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 390 insertions(+) create mode 100644 t/discovery/consul_dump.t diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t new file mode 100644 index 000000000000..c1dbb0b7392e --- /dev/null +++ b/t/discovery/consul_dump.t @@ -0,0 +1,390 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + + +add_block_preprocessor(sub { + my ($block) = @_; + + my $http_config = $block->http_config // <<_EOC_; + + server { + listen 30511; + + location /hello { + content_by_lua_block { + ngx.say("server 1") + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +run_tests(); + +__DATA__ + +=== TEST 1: prepare nodes +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "DELETE /v1/kv/upstreams/?recurse=true", + "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", +] +--- response_body eval +[ + 'true', + 'true', + 'true', +] + + + +=== TEST 2: show dump services +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + dump: + path: "consul.dump" + load_on_init: true +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + ngx.sleep(2) + + local code, body, res = t.test('/v1/discovery/consul/show_dump_file', + ngx.HTTP_GET) + local entity = json.decode(res) + ngx.say(json.encode(entity.services)) + } + } +--- timeout: 3 +--- request +GET /t +--- response_body +{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]} + + + +=== TEST 3: prepare dump file for next test +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + dump: + path: "/tmp/consul.dump" + load_on_init: true +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- response_body +server 1 + + + +=== TEST 4: clean registered nodes +--- config +location /v1/kv { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "DELETE /v1/kv/upstreams/?recurse=true", +] +--- response_body eval +[ + 'true' +] + + + +=== TEST 5: test load dump on init +Configure the invalid consul server addr, and loading the last test 3 generated /tmp/consul.dump file into memory when initializing +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul.dump" + load_on_init: true +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- response_body +server 1 +--- error_log +connect consul + + + +=== TEST 6: delete dump file +--- config + location /t { + content_by_lua_block { + local util = require("apisix.cli.util") + local succ, err = util.execute_cmd("rm -f /tmp/consul.dump") + ngx.say(succ and "success" or err) + } + } +--- request +GET /t +--- response_body +success + + + +=== TEST 7: miss load dump on init +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul.dump" + load_on_init: true +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +connect consul +fetch nodes failed +failed to set upstream + + + +=== TEST 8: prepare expired dump file +--- config + location /t { + content_by_lua_block { + local util = require("apisix.cli.util") + local json = require("toolkit.json") + + local applications = json.decode('{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]}') + local entity = { + services = applications, + last_update = ngx.time(), + expire = 10, + } + local succ, err = util.write_file("/tmp/consul.dump", json.encode(entity)) + + ngx.sleep(2) + ngx.say(succ and "success" or err) + } + } +--- timeout: 3 +--- request +GET /t +--- response_body +success + + + +=== TEST 9: unexpired dump +test load unexpired /tmp/consul.dump file generated by upper test when initializing + when initializing +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul.dump" + load_on_init: true + expire: 5 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- response_body +server 1 +--- error_log +connect consul + + + +=== TEST 10: expired dump +test load expired ( by check: (dump_file.last_update + dump.expire) < ngx.time ) ) /tmp/consul.dump file generated by upper test when initializing + when initializing +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + consul: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul.dump" + load_on_init: true + expire: 1 +#END +--- apisix_yaml +routes: + - + uri: /* + upstream: + service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + discovery_type: consul + type: roundrobin +#END +--- request +GET /hello +--- error_code: 503 +--- error_log +dump file: /tmp/consul.dump had expired, ignored it + + + +=== TEST 11: delete dump file +--- config + location /t { + content_by_lua_block { + local util = require("apisix.cli.util") + local succ, err = util.execute_cmd("rm -f /tmp/consul.dump") + ngx.say(succ and "success" or err) + } + } +--- request +GET /t +--- response_body +success + + + +=== TEST 12: dump file inexistence +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +discovery: + consul: + servers: + - "http://127.0.0.1:38500" + dump: + path: "/tmp/consul.dump" +#END +--- request +GET /v1/discovery/consul/show_dump_file +--- error_code: 503 +--- error_log +connect consul + + + +=== TEST 13: no dump config +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +discovery: + consul: + servers: + - "http://127.0.0.1:38500" +#END +--- request +GET /v1/discovery/consul/show_dump_file +--- error_code: 503 +--- error_log +connect consul From 793a365e51518cb150c0c24a070bc16fbc35213f Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 23 Nov 2022 06:06:43 +0000 Subject: [PATCH 07/27] feat(discovery): add consul discovery --- t/discovery/consul.t | 35 +++++++++++++++++------------------ t/discovery/consul_dump.t | 8 ++++---- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/t/discovery/consul.t b/t/discovery/consul.t index 70ee221ec52d..65dea0d23d25 100644 --- a/t/discovery/consul.t +++ b/t/discovery/consul.t @@ -90,9 +90,8 @@ discovery: servers: - "http://127.0.0.1:8500" - "http://127.0.0.1:8600" - prefix: "upstreams" - skip_keys: - - "upstreams/unused_api/" + skip_services: + - "service_a" timeout: connect: 1000 read: 1000 @@ -114,15 +113,15 @@ run_tests(); __DATA__ -=== TEST 1: prepare consul kv register nodes +=== TEST 1: prepare consul catalog register nodes --- config location /consul1 { - rewrite ^/consul1/(.*) /v1/kv/$1 break; + rewrite ^/consul1/(.*) /v1/catalog/services/$1 break; proxy_pass http://127.0.0.1:8500; } location /consul2 { - rewrite ^/consul2/(.*) /v1/kv/$1 break; + rewrite ^/consul2/(.*) /v1/catalog/services/$1 break; proxy_pass http://127.0.0.1:8600; } --- pipelined_requests eval @@ -146,7 +145,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8500/v1/catalog/services/ discovery_type: consul type: roundrobin #END @@ -172,7 +171,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8600/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8600/v1/catalog/services/ discovery_type: consul type: roundrobin #END @@ -210,7 +209,7 @@ routes: - uri: /hello upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8500/v1/catalog/services/ discovery_type: consul type: roundrobin #END @@ -231,7 +230,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8600/v1/kv/upstreams/deleted_keys/ + service_name: http://127.0.0.1:8600/v1/catalog/services/dele_keys/ discovery_type: consul type: roundrobin #END @@ -263,8 +262,8 @@ discovery: servers: - "http://127.0.0.1:8600" prefix: "upstreams" - skip_keys: - - "upstreams/webpages/" + skip_services: + - "service_a" default_service: host: "127.0.0.1" port: 20999 @@ -278,7 +277,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8600/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8600/v1/catalog/services/ discovery_type: consul type: roundrobin #END @@ -297,12 +296,12 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8500/v1/catalog/services/ discovery_type: consul type: roundrobin #END --- config -location /v1/kv { +location /v1/catalog/services { proxy_pass http://127.0.0.1:8500; } location /sleep { @@ -369,7 +368,7 @@ location /sleep { === TEST 8: prepare healthy and unhealthy nodes --- config -location /v1/kv { +location /v1/catalog/services { proxy_pass http://127.0.0.1:8500; } --- request eval @@ -397,7 +396,7 @@ routes: upstream_id: 1 upstreams: - - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8500/v1/catalog/services/ discovery_type: consul type: roundrobin id: 1 @@ -545,7 +544,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8501/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8501/v1/catalog/services/ discovery_type: consul type: roundrobin #END diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index c1dbb0b7392e..20a6eb3edf02 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -48,7 +48,7 @@ __DATA__ === TEST 1: prepare nodes --- config -location /v1/kv { +location /v1/catalog/services { proxy_pass http://127.0.0.1:8500; } --- request eval @@ -94,7 +94,7 @@ discovery: --- request GET /t --- response_body -{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]} +{"http://127.0.0.1:8500/v1/catalog/services/":[{"host":"127.0.0.1","port":30511,"weight":1}]} @@ -120,7 +120,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8500/v1/catalog/services/ discovery_type: consul type: roundrobin #END @@ -325,7 +325,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: http://127.0.0.1:8500/v1/catalog/services/ discovery_type: consul type: roundrobin #END From 05c23fe7e0d1dea4bd8059dcbf95c5b106ec843a Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 23 Nov 2022 07:14:14 +0000 Subject: [PATCH 08/27] feat: add consul discovery module --- apisix/discovery/consul/init.lua | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index d9124067b0d8..fab3d547bfe4 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -222,8 +222,10 @@ function _M.connect(premature, consul_server, retry_delay) or ((watch_result ~= nil and watch_result.status ~= 200) and watch_result.status) if watch_error_info then - log.error("connect consul: ", consul_server.consul_server_url, " by sub url: ", consul_server.consul_sub_url, - ", got watch result: ", json_delay_encode(watch_result, true), ", with error: ", watch_error_info) + log.error("connect consul: ", consul_server.consul_server_url, + " by sub url: ", consul_server.consul_sub_url, + ", got watch result: ", json_delay_encode(watch_result, true), + ", with error: ", watch_error_info) if not retry_delay then retry_delay = 1 @@ -249,7 +251,8 @@ function _M.connect(premature, consul_server, retry_delay) -- fetch all services info local result, err = consul_client:get(consul_server.consul_sub_url) - local error_info = (err ~= nil and err) or ((result ~= nil and result.status ~= 200) and result.status) + local error_info = (err ~= nil and err) or + ((result ~= nil and result.status ~= 200) and result.status) if error_info then log.error("connect consul: ", consul_server.consul_server_url, From c3aa851584176dc2ce2a22cc06b7832e723ba510 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 23 Nov 2022 08:23:38 +0000 Subject: [PATCH 09/27] feat: add consul discovery --- apisix/discovery/consul/init.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index fab3d547bfe4..4b5d9489e696 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -278,7 +278,7 @@ function _M.connect(premature, consul_server, retry_delay) consul_server.default_args.index = watch_result.headers['X-Consul-Index'] end -- decode body, decode json, update service, error handling - if result.body and next(result.body) then + if result.body then log.notice("server_name: ", consul_server.consul_server_url, ", header: ", core.json.encode(result.headers, true), ", body: ", core.json.encode(result.body, true)) From 8c89e09dd6af03af2f937feac8b61d1793b0c9cd Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 23 Nov 2022 23:14:07 +0800 Subject: [PATCH 10/27] feat: add consul discovery --- t/discovery/consul_dump.t | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 20a6eb3edf02..2a7b894ca027 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -48,19 +48,22 @@ __DATA__ === TEST 1: prepare nodes --- config -location /v1/catalog/services { +location /v1/agent { proxy_pass http://127.0.0.1:8500; } --- request eval [ - "DELETE /v1/kv/upstreams/?recurse=true", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", + "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_b1", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":8002,\"Meta\":{\"service_a_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval [ - 'true', - 'true', - 'true', + '', + '', + '', + '', ] @@ -94,8 +97,7 @@ discovery: --- request GET /t --- response_body -{"http://127.0.0.1:8500/v1/catalog/services/":[{"host":"127.0.0.1","port":30511,"weight":1}]} - +{"service_a":[{"host":"127.0.0.1","port":30511,"weight":1}],"service_b":[{"host":"127.0.0.1","port":8002,"weight":1}]} === TEST 3: prepare dump file for next test @@ -120,7 +122,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -133,16 +135,18 @@ server 1 === TEST 4: clean registered nodes --- config -location /v1/kv { +location /v1/agent { proxy_pass http://127.0.0.1:8500; } --- request eval [ - "DELETE /v1/kv/upstreams/?recurse=true", + "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_b1", ] --- response_body eval [ - 'true' + '', + '' ] @@ -170,7 +174,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -221,7 +225,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -242,7 +246,7 @@ failed to set upstream local util = require("apisix.cli.util") local json = require("toolkit.json") - local applications = json.decode('{"http://127.0.0.1:8500/v1/kv/upstreams/webpages/":[{"host":"127.0.0.1","port":30511,"weight":1}]}') + local applications = json.decode('{"service_a":[{"host":"127.0.0.1","port":30511,"weight":1}]}') local entity = { services = applications, last_update = ngx.time(), @@ -287,7 +291,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -325,7 +329,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END From cd9d9fc33ccb1dae8a819cfc70a9fca1a3855660 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 23 Nov 2022 23:20:37 +0800 Subject: [PATCH 11/27] feat: add consul discovery --- ci/pod/docker-compose.first.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/pod/docker-compose.first.yml b/ci/pod/docker-compose.first.yml index a13ad3cf1586..62ef7a328c16 100644 --- a/ci/pod/docker-compose.first.yml +++ b/ci/pod/docker-compose.first.yml @@ -33,7 +33,7 @@ services: restart: unless-stopped ports: - "8500:8500" - command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-client", "0.0.0.0", "-log-level", "info", "-data-dir=/consul/data" ] + command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-client", "0.0.0.0", "-log-level", "info", "-data-dir=/consul/data", "-enable-script-checks" ] networks: consul_net: @@ -42,7 +42,7 @@ services: restart: unless-stopped ports: - "8600:8500" - command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-client", "0.0.0.0", "-log-level", "info", "-data-dir=/consul/data" ] + command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-client", "0.0.0.0", "-log-level", "info", "-data-dir=/consul/data", "-enable-script-checks" ] networks: consul_net: From 9822bc4931bd1203605f989cd135605ebe517626 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Thu, 24 Nov 2022 12:10:33 +0800 Subject: [PATCH 12/27] feat: update unit test --- apisix/discovery/consul/init.lua | 7 +- t/discovery/consul.t | 158 ++++++++++--------------------- t/discovery/consul_dump.t | 2 +- 3 files changed, 52 insertions(+), 115 deletions(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index 4b5d9489e696..6cdbba322859 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -313,12 +313,9 @@ local function format_consul_params(consul_conf) local args if consul_conf.keepalive == false then - args = { - recurse = true, - } + args = {} elseif consul_conf.keepalive then args = { - recurse = true, wait = consul_conf.timeout.wait, --blocked wait!=0; unblocked by wait=0 index = 0, } @@ -405,7 +402,7 @@ function _M.init_worker() return end - if server.keepalive == true then + if server.keepalive == false then ngx_timer_every(server.fetch_interval, _M.connect, server) end end diff --git a/t/discovery/consul.t b/t/discovery/consul.t index 65dea0d23d25..1db9f7180363 100644 --- a/t/discovery/consul.t +++ b/t/discovery/consul.t @@ -81,6 +81,10 @@ _EOC_ our $yaml_config = <<_EOC_; apisix: node_listen: 1984 + enable_control: true + control: + ip: 127.0.0.1 + port: 9090 deployment: role: data_plane role_data_plane: @@ -91,7 +95,7 @@ discovery: - "http://127.0.0.1:8500" - "http://127.0.0.1:8600" skip_services: - - "service_a" + - "service_c" timeout: connect: 1000 read: 1000 @@ -116,7 +120,7 @@ __DATA__ === TEST 1: prepare consul catalog register nodes --- config location /consul1 { - rewrite ^/consul1/(.*) /v1/catalog/services/$1 break; + rewrite ^/consul1/(.*) /v1/agent/service/$1 break; proxy_pass http://127.0.0.1:8500; } @@ -126,16 +130,15 @@ location /consul2 { } --- pipelined_requests eval [ - "DELETE /consul1/upstreams/webpages/?recurse=true", - "DELETE /consul2/upstreams/webpages/?recurse=true", - "PUT /consul1/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", - "PUT /consul1/upstreams/webpages/127.0.0.1:30512\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", - "PUT /consul2/upstreams/webpages/127.0.0.1:30513\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", - "PUT /consul2/upstreams/webpages/127.0.0.1:30514\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /consul1/deregister/service_a1", + "PUT /consul1/deregister/service_b1", + "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /consul1/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /consul1/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval -["true", "true", "true", "true", "true", "true"] - +["", "", "", "", "", "", "", ""] === TEST 2: test consul server 1 @@ -145,7 +148,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -171,7 +174,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8600/v1/catalog/services/ + service_name: service_b discovery_type: consul type: roundrobin #END @@ -209,7 +212,7 @@ routes: - uri: /hello upstream: - service_name: http://127.0.0.1:8500/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -221,8 +224,7 @@ qr/server [1-2]/ -=== TEST 5: test invalid service name -sometimes the consul key maybe deleted by mistake +=== TEST 5: test invalid service name sometimes the consul key maybe deleted by mistake --- yaml_config eval: $::yaml_config --- apisix_yaml @@ -230,7 +232,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8600/v1/catalog/services/dele_keys/ + service_name: service_c discovery_type: consul type: roundrobin #END @@ -249,7 +251,7 @@ routes: === TEST 6: test skip keys -skip some keys, return default nodes, get response: missing consul services +skip some services, return default nodes, get response: missing consul services --- yaml_config apisix: node_listen: 1984 @@ -277,7 +279,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8600/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END @@ -296,12 +298,12 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END --- config -location /v1/catalog/services { +location /v1/agent { proxy_pass http://127.0.0.1:8500; } location /sleep { @@ -315,10 +317,9 @@ location /sleep { --- timeout: 6 --- request eval [ - "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30511", - "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30512", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30513\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30514\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "GET /sleep", "GET /hello?random1", @@ -326,10 +327,9 @@ location /sleep { "GET /hello?random3", "GET /hello?random4", - "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30513", - "DELETE /v1/kv/upstreams/webpages/127.0.0.1:30514", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30512\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "GET /sleep?sec=5", "GET /hello?random1", @@ -340,10 +340,9 @@ location /sleep { ] --- response_body_like eval [ - qr/true/, - qr/true/, - qr/true/, - qr/true/, + qr//, + qr//, + qr//, qr/ok\n/, qr/server [3-4]\n/, @@ -351,10 +350,9 @@ location /sleep { qr/server [3-4]\n/, qr/server [3-4]\n/, - qr/true/, - qr/true/, - qr/true/, - qr/true/, + qr//, + qr//, + qr//, qr/ok\n/, qr/server [1-2]\n/, @@ -368,97 +366,39 @@ location /sleep { === TEST 8: prepare healthy and unhealthy nodes --- config -location /v1/catalog/services { +location /v1/agent { proxy_pass http://127.0.0.1:8500; } --- request eval [ - "DELETE /v1/kv/upstreams/webpages/?recurse=true", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", - "PUT /v1/kv/upstreams/webpages/127.0.0.2:1988\n" . "{\"weight\": 1, \"max_fails\": 1, \"fail_timeout\": 1}", + "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.2\",\"Port\":1988,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval [ - 'true', - 'true', - 'true', + '', + '', + '', ] -=== TEST 9: test health checker ---- yaml_config eval: $::yaml_config ---- apisix_yaml -routes: - - - uris: - - /hello - upstream_id: 1 -upstreams: - - - service_name: http://127.0.0.1:8500/v1/catalog/services/ - discovery_type: consul - type: roundrobin - id: 1 - checks: - active: - http_path: "/hello" - healthy: - interval: 1 - successes: 1 - unhealthy: - interval: 1 - http_failures: 1 -#END ---- config - location /thc { - content_by_lua_block { - local json = require("toolkit.json") - local t = require("lib.test_admin") - local http = require "resty.http" - local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" - local httpc = http.new() - httpc:request_uri(uri, {method = "GET"}) - ngx.sleep(3) - - local code, body, res = t.test('/v1/healthcheck', - ngx.HTTP_GET) - res = json.decode(res) - table.sort(res[1].nodes, function(a, b) - return a.host < b.host - end) - ngx.say(json.encode(res)) - - local code, body, res = t.test('/v1/healthcheck/upstreams/1', - ngx.HTTP_GET) - res = json.decode(res) - table.sort(res.nodes, function(a, b) - return a.host < b.host - end) - ngx.say(json.encode(res)) - } - } ---- request -GET /thc ---- response_body -[{"healthy_nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1}],"name":"upstream#/upstreams/1","nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1},{"host":"127.0.0.2","port":1988,"priority":0,"weight":1}],"src_id":"1","src_type":"upstreams"}] -{"healthy_nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1}],"name":"upstream#/upstreams/1","nodes":[{"host":"127.0.0.1","port":30511,"priority":0,"weight":1},{"host":"127.0.0.2","port":1988,"priority":0,"weight":1}],"src_id":"1","src_type":"upstreams"} ---- ignore_error_log === TEST 10: clean nodes --- config -location /v1/kv { +location /v1/agent { proxy_pass http://127.0.0.1:8500; } --- request eval [ - "DELETE /v1/kv/upstreams/webpages/?recurse=true" + "PUT /v1/agent/service/deregister/service_a1", ] --- response_body eval [ - 'true' + '' ] @@ -486,12 +426,12 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8500/v1/kv/upstreams/webpages/ + service_name: service_a discovery_type: consul type: roundrobin #END --- config -location /v1/kv { +location /v1/agent { proxy_pass http://127.0.0.1:8500; } location /sleep { @@ -506,14 +446,14 @@ location /sleep { --- request eval [ "GET /hello", - "PUT /v1/kv/upstreams/webpages/127.0.0.1:30511\n" . "{\"weight\": 1, \"max_fails\": 2, \"fail_timeout\": 1}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "GET /sleep?sec=5", "GET /hello", ] --- response_body_like eval [ qr/missing consul services\n/, - qr/true/, + qr//, qr/ok\n/, qr/server 1\n/ ] @@ -544,7 +484,7 @@ routes: - uri: /* upstream: - service_name: http://127.0.0.1:8501/v1/catalog/services/ + service_name: service_a discovery_type: consul type: roundrobin #END diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 2a7b894ca027..3a6e363b0b37 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -56,7 +56,7 @@ location /v1/agent { "PUT /v1/agent/service/deregister/service_a1", "PUT /v1/agent/service/deregister/service_b1", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":8002,\"Meta\":{\"service_a_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":8002,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval [ From e3e210b4d28c3716bbe022cf5c2f591f2f3f9135 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Thu, 24 Nov 2022 14:58:26 +0800 Subject: [PATCH 13/27] feat: add consul register file --- docs/en/latest/discovery/consul.md | 296 +++++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 docs/en/latest/discovery/consul.md diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md new file mode 100644 index 000000000000..d4e14d4832f5 --- /dev/null +++ b/docs/en/latest/discovery/consul.md @@ -0,0 +1,296 @@ +--- +title: consul +--- + + + +## Summary + +APACHE APISIX supports Consul as a service discovery + +## Configuration for discovery client + +### Configuration for Consul + +First of all, we need to add following configuration in `conf/config.yaml` : + +```yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + - "http://127.0.0.1:8600" + skip_services: # if you need to skip special keys + - "service_a" + timeout: + connect: 1000 # default 2000 ms + read: 1000 # default 2000 ms + wait: 60 # default 60 sec + weight: 1 # default 1 + fetch_interval: 5 # default 3 sec, only take effect for keepalive: false way + keepalive: true # default true, use the long pull way to query consul servers + default_service: # you can define default service when missing hit + host: "127.0.0.1" + port: 20999 + metadata: + fail_timeout: 1 # default 1 ms + weight: 1 # default 1 + max_fails: 1 # default 1 + dump: # if you need, when registered nodes updated can dump into file + path: "logs/consul.dump" + expire: 2592000 # unit sec, here is 30 day +``` + +And you can config it in short by default value: + +```yaml +discovery: + consul: + servers: + - "http://127.0.0.1:8500" +``` + +The `keepalive` has two optional values: + +- `true`, default and recommend value, use the long pull way to query consul servers +- `false`, not recommend, it would use the short pull way to query consul servers, then you can set the `fetch_interval` for fetch interval + +#### Dump Data + +When we need reload `apisix` online, as the `consul` module maybe loads data from CONSUL slower than load routes from ETCD, and would get the log at the moment before load successfully from consul: + +``` + http_access_phase(): failed to set upstream: no valid upstream node +``` + +So, we import the `dump` function for `consul` module. When reload, would load the dump file before from consul; when the registered nodes in consul been updated, would dump the upstream nodes into file automatically. + +The `dump` has three optional values now: + +- `path`, the dump file save path + - support relative path, eg: `logs/consul.dump` + - support absolute path, eg: `/tmp/consul.dump` + - make sure the dump file's parent path exist + - make sure the `apisix` has the dump file's read-write access permission,eg: add below config in `conf/config.yaml` + ```yaml + nginx_config: # config for render the template to generate nginx.conf + user: root # specifies the execution user of the worker process. + ``` +- `load_on_init`, default value is `true` + - if `true`, just try to load the data from the dump file before loading data from consul when starting, does not care the dump file exists or not + - if `false`, ignore loading data from the dump file + - Whether `true` or `false`, we don't need to prepare a dump file for apisix at anytime +- `expire`, unit sec, avoiding load expired dump data when load + - default `0`, it is unexpired forever + - recommend 2592000, which is 30 days(equals 3600 \* 24 \* 30) + +### Register Http API Services + +Now, register nodes into consul: + +```shell +curl -X PUT 'http://127.0.0.1:8500/v1/agent/service/register' \ +-d '{ + "ID": "service_a1", + "Name": "service_a", + "Tags": ["primary", "v1"], + "Address": "127.0.0.1", + "Port": 8000, + "Meta": { + "service_a_version": "4.0" + }, + "EnableTagOverride": false, + "Weights": { + "Passing": 10, + "Warning": 1 + } +}' + +curl -X PUT 'http://127.0.0.1:8500/v1/agent/service/register' \ +-d '{ + "ID": "service_a1", + "Name": "service_a", + "Tags": ["primary", "v1"], + "Address": "127.0.0.1", + "Port": 8002, + "Meta": { + "service_a_version": "4.0" + }, + "EnableTagOverride": false, + "Weights": { + "Passing": 10, + "Warning": 1 + } +}' +``` + +In some case, same service name exist in different consul servers. +To avoid confusion, use the full consul key url path as service name in practice. + +### Upstream setting + +Here is an example of routing a request with a URL of "/*" to a service which named "service_a" and use consul discovery client in the registry : + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -i -d ' +{ + "uri": "/*", + "upstream": { + "service_name": "service_a", + "type": "roundrobin", + "discovery_type": "consul" + } +}' +``` + +The format response as below: + +```json +{ + "key": "/apisix/routes/1", + "value": { + "uri": "/*", + "priority": 0, + "id": "1", + "upstream": { + "scheme": "http", + "type": "roundrobin", + "hash_on": "vars", + "discovery_type": "consul", + "service_name": "service_a", + "pass_host": "pass" + }, + "create_time": 1669267329, + "status": 1, + "update_time": 1669267329 + } +} +``` + +You could find more usage in the `apisix/t/discovery/consul.t` file. + +## Debugging API + +It also offers control api for debugging. + +### Memory Dump API + +```shell +GET /v1/discovery/consul/dump +``` + +For example: + +```shell +# curl http://127.0.0.1:9090/v1/discovery/consul/dump | jq +{ + "config": { + "fetch_interval": 3, + "timeout": { + "wait": 60, + "connect": 6000, + "read": 6000 + }, + "weight": 1, + "servers": [ + "http://172.19.5.30:8500", + "http://172.19.5.31:8500" + ], + "keepalive": true, + "default_service": { + "host": "172.19.5.11", + "port": 8899, + "metadata": { + "fail_timeout": 1, + "weight": 1, + "max_fails": 1 + } + }, + "skip_services": [ + "service_d" + ] + }, + "services": { + "service_a": [ + { + "host": "127.0.0.1", + "port": 30513, + "weight": 1 + }, + { + "host": "127.0.0.1", + "port": 30514, + "weight": 1 + } + ], + "service_b": [ + { + "host": "172.19.5.51", + "port": 50051, + "weight": 1 + } + ], + "service_c": [ + { + "host": "127.0.0.1", + "port": 30511, + "weight": 1 + }, + { + "host": "127.0.0.1", + "port": 30512, + "weight": 1 + } + ] + } +} +``` + +### Show Dump File API + +It offers another control api for dump file view now. Maybe would add more api for debugging in future. + +```shell +GET /v1/discovery/consul/show_dump_file +``` + +For example: + +```shell +curl http://127.0.0.1:9090/v1/discovery/consul/show_dump_file | jq +{ + "services": { + "service_a": [ + { + "host": "172.19.5.12", + "port": 8000, + "weight": 120 + }, + { + "host": "172.19.5.13", + "port": 8000, + "weight": 120 + } + ] + }, + "expire": 0, + "last_update": 1615877468 +} +``` From 4e695b1ed4e76f4b1702e1df051765aec0048418 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Thu, 24 Nov 2022 08:15:12 +0000 Subject: [PATCH 14/27] feat: fix ci error --- apisix/discovery/consul/init.lua | 10 ---------- docs/en/latest/config.json | 1 + docs/en/latest/discovery/consul.md | 8 ++++---- t/discovery/consul.t | 10 ++++------ t/discovery/consul_dump.t | 1 + 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index 6cdbba322859..e73caebd5e5a 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -86,16 +86,6 @@ local function parse_instance(node) if service_name and skip_service_map[service_name] then return false end - - -- if host is empty, get host and port from TaggedAddresses.wan - if not host then - local tagged_addresses = node.TaggedAddresses - if tagged_addresses then - host = tagged_addresses.wan.address - port = tagged_addresses.wan.port - end - end - return true, host, tonumber(port), "", service_name end diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index ae6dd95d4296..e7b5a5ba233c 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -264,6 +264,7 @@ "discovery", "discovery/dns", "discovery/consul_kv", + "discovery/consul", "discovery/nacos", "discovery/eureka", "discovery/control-plane-service-discovery", diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md index d4e14d4832f5..39eca771f1d7 100644 --- a/docs/en/latest/discovery/consul.md +++ b/docs/en/latest/discovery/consul.md @@ -89,10 +89,10 @@ The `dump` has three optional values now: - support absolute path, eg: `/tmp/consul.dump` - make sure the dump file's parent path exist - make sure the `apisix` has the dump file's read-write access permission,eg: add below config in `conf/config.yaml` - ```yaml - nginx_config: # config for render the template to generate nginx.conf - user: root # specifies the execution user of the worker process. - ``` +```yaml +nginx_config: # config for render the template to generate nginx.conf + user: root # specifies the execution user of the worker process. +``` - `load_on_init`, default value is `true` - if `true`, just try to load the data from the dump file before loading data from consul when starting, does not care the dump file exists or not - if `false`, ignore loading data from the dump file diff --git a/t/discovery/consul.t b/t/discovery/consul.t index 1db9f7180363..d6c127b9ef09 100644 --- a/t/discovery/consul.t +++ b/t/discovery/consul.t @@ -141,6 +141,7 @@ location /consul2 { ["", "", "", "", "", "", "", ""] + === TEST 2: test consul server 1 --- yaml_config eval: $::yaml_config --- apisix_yaml @@ -384,10 +385,7 @@ location /v1/agent { - - - -=== TEST 10: clean nodes +=== TEST 9: clean nodes --- config location /v1/agent { proxy_pass http://127.0.0.1:8500; @@ -403,7 +401,7 @@ location /v1/agent { -=== TEST 11: test consul short connect type +=== TEST 10: test consul short connect type --- yaml_config apisix: node_listen: 1984 @@ -461,7 +459,7 @@ location /sleep { -=== TEST 12: retry when Consul can't be reached (long connect type) +=== TEST 11: retry when Consul can't be reached (long connect type) --- yaml_config apisix: node_listen: 1984 diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 3a6e363b0b37..27536748b2f3 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -100,6 +100,7 @@ GET /t {"service_a":[{"host":"127.0.0.1","port":30511,"weight":1}],"service_b":[{"host":"127.0.0.1","port":8002,"weight":1}]} + === TEST 3: prepare dump file for next test --- yaml_config apisix: From 369430537d8d77347b2fd049b7275f308359dd13 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Thu, 24 Nov 2022 08:25:59 +0000 Subject: [PATCH 15/27] feat: fix unit test error --- t/discovery/consul_dump.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 27536748b2f3..2e01b638f721 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -116,7 +116,7 @@ discovery: - "http://127.0.0.1:8500" dump: path: "/tmp/consul.dump" - load_on_init: true + load_on_init: false #END --- apisix_yaml routes: From abdc520ab97307962643eb51891734ed44815e15 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Thu, 24 Nov 2022 08:59:41 +0000 Subject: [PATCH 16/27] feat: corect variable name --- apisix/discovery/consul/init.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index e73caebd5e5a..3585e3edb037 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -213,7 +213,7 @@ function _M.connect(premature, consul_server, retry_delay) and watch_result.status) if watch_error_info then log.error("connect consul: ", consul_server.consul_server_url, - " by sub url: ", consul_server.consul_sub_url, + " by sub url: ", consul_server.consul_watch_sub_url, ", got watch result: ", json_delay_encode(watch_result, true), ", with error: ", watch_error_info) From 5490ade6fcc43412e3f21f307b879fd5d8d07474 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Fri, 25 Nov 2022 05:18:00 +0000 Subject: [PATCH 17/27] feat: fix ci error --- docs/en/latest/discovery/consul.md | 54 ++++++++++++++++-------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md index 39eca771f1d7..9f1dde42f603 100644 --- a/docs/en/latest/discovery/consul.md +++ b/docs/en/latest/discovery/consul.md @@ -89,10 +89,12 @@ The `dump` has three optional values now: - support absolute path, eg: `/tmp/consul.dump` - make sure the dump file's parent path exist - make sure the `apisix` has the dump file's read-write access permission,eg: add below config in `conf/config.yaml` + ```yaml nginx_config: # config for render the template to generate nginx.conf user: root # specifies the execution user of the worker process. ``` + - `load_on_init`, default value is `true` - if `true`, just try to load the data from the dump file before loading data from consul when starting, does not care the dump file exists or not - if `false`, ignore loading data from the dump file @@ -108,36 +110,36 @@ Now, register nodes into consul: ```shell curl -X PUT 'http://127.0.0.1:8500/v1/agent/service/register' \ -d '{ - "ID": "service_a1", - "Name": "service_a", - "Tags": ["primary", "v1"], - "Address": "127.0.0.1", - "Port": 8000, - "Meta": { - "service_a_version": "4.0" - }, - "EnableTagOverride": false, - "Weights": { - "Passing": 10, - "Warning": 1 - } + "ID": "service_a1", + "Name": "service_a", + "Tags": ["primary", "v1"], + "Address": "127.0.0.1", + "Port": 8000, + "Meta": { + "service_a_version": "4.0" + }, + "EnableTagOverride": false, + "Weights": { + "Passing": 10, + "Warning": 1 + } }' curl -X PUT 'http://127.0.0.1:8500/v1/agent/service/register' \ -d '{ - "ID": "service_a1", - "Name": "service_a", - "Tags": ["primary", "v1"], - "Address": "127.0.0.1", - "Port": 8002, - "Meta": { - "service_a_version": "4.0" - }, - "EnableTagOverride": false, - "Weights": { - "Passing": 10, - "Warning": 1 - } + "ID": "service_a1", + "Name": "service_a", + "Tags": ["primary", "v1"], + "Address": "127.0.0.1", + "Port": 8002, + "Meta": { + "service_a_version": "4.0" + }, + "EnableTagOverride": false, + "Weights": { + "Passing": 10, + "Warning": 1 + } }' ``` From f916d3e1cb81c664831fa0faa8933bdc2fd20e21 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Fri, 25 Nov 2022 05:31:56 +0000 Subject: [PATCH 18/27] feat: fix cr, add comment --- apisix/discovery/consul/init.lua | 2 +- docs/en/latest/discovery/consul.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index 3585e3edb037..b16510da5f6d 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -51,7 +51,7 @@ local function discovery_consul_callback(data, event, source, pid) all_services = data log.notice("update local variable all_services, event is: ", event, "source: ", source, "server pid:", pid, - ", all services: ", core.json.encode(all_services, true)) + ", all services: ", json_delay_encode(all_services, true)) end diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md index 9f1dde42f603..2bb12afc278e 100644 --- a/docs/en/latest/discovery/consul.md +++ b/docs/en/latest/discovery/consul.md @@ -36,8 +36,8 @@ discovery: consul: servers: - "http://127.0.0.1:8500" - - "http://127.0.0.1:8600" - skip_services: # if you need to skip special keys + - "http://127.0.0.1:8600" # `http://127.0.0.1:8500` and `http://127.0.0.1:8600` are different clusters + skip_services: # if you need to skip special keys - "service_a" timeout: connect: 1000 # default 2000 ms @@ -46,7 +46,7 @@ discovery: weight: 1 # default 1 fetch_interval: 5 # default 3 sec, only take effect for keepalive: false way keepalive: true # default true, use the long pull way to query consul servers - default_service: # you can define default service when missing hit + default_service: # you can define default service when missing hit host: "127.0.0.1" port: 20999 metadata: @@ -55,7 +55,7 @@ discovery: max_fails: 1 # default 1 dump: # if you need, when registered nodes updated can dump into file path: "logs/consul.dump" - expire: 2592000 # unit sec, here is 30 day + expire: 2592000 # unit sec, here is 30 day ``` And you can config it in short by default value: From 0f73b7789876b73494e6287729f98ccd4b94b5e9 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Sun, 27 Nov 2022 00:34:38 +0800 Subject: [PATCH 19/27] feat: add heathycheck unit test --- t/discovery/consul.t | 85 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/t/discovery/consul.t b/t/discovery/consul.t index d6c127b9ef09..8797f8912fc4 100644 --- a/t/discovery/consul.t +++ b/t/discovery/consul.t @@ -132,10 +132,12 @@ location /consul2 { [ "PUT /consul1/deregister/service_a1", "PUT /consul1/deregister/service_b1", + "PUT /consul1/deregister/service_a2", + "PUT /consul1/deregister/service_b2", "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /consul1/register\n" . "{\"ID\":\"service_a2\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /consul1/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /consul1/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /consul1/register\n" . "{\"ID\":\"service_b2\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval ["", "", "", "", "", "", "", ""] @@ -319,8 +321,9 @@ location /sleep { --- request eval [ "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_a2", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a2\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "GET /sleep", "GET /hello?random1", @@ -329,8 +332,9 @@ location /sleep { "GET /hello?random4", "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_a2", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a2\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "GET /sleep?sec=5", "GET /hello?random1", @@ -344,6 +348,7 @@ location /sleep { qr//, qr//, qr//, + qr//, qr/ok\n/, qr/server [3-4]\n/, @@ -354,6 +359,7 @@ location /sleep { qr//, qr//, qr//, + qr//, qr/ok\n/, qr/server [1-2]\n/, @@ -373,14 +379,16 @@ location /v1/agent { --- request eval [ "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_a2", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.2\",\"Port\":1988,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a2\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.2\",\"Port\":1988,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval [ '', '', '', + '', ] @@ -393,10 +401,12 @@ location /v1/agent { --- request eval [ "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_a2", ] --- response_body eval [ - '' + '', + '', ] @@ -505,3 +515,66 @@ qr/retry connecting consul after \d seconds/ --- grep_error_log_out retry connecting consul after 1 seconds retry connecting consul after 4 seconds + + + +=== TEST 12: test health checker +--- yaml_config eval: $::yaml_config +--- apisix_yaml +routes: + - + uris: + - /hello + upstream_id: 1 +upstreams: + - + service_name: service_b + discovery_type: consul + type: roundrobin + id: 1 + checks: + active: + http_path: "/hello" + healthy: + interval: 1 + successes: 1 + unhealthy: + interval: 1 + http_failures: 1 +#END +--- config + location /thc { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + httpc:request_uri(uri, {method = "GET"}) + ngx.sleep(3) + + local code, body, res = t.test('/v1/healthcheck', + ngx.HTTP_GET) + res = json.decode(res) + local nodes = res[1].nodes + table.sort(nodes, function(a, b) + return a.port < b.port + end) + ngx.say(json.encode(nodes)) + + local code, body, res = t.test('/v1/healthcheck/upstreams/1', + ngx.HTTP_GET) + res = json.decode(res) + nodes = res.nodes + table.sort(nodes, function(a, b) + return a.port < b.port + end) + ngx.say(json.encode(nodes)) + } + } +--- request +GET /thc +--- response_body +[{"host":"127.0.0.1","port":30513,"priority":0,"weight":1},{"host":"127.0.0.1","port":30514,"priority":0,"weight":1}] +[{"host":"127.0.0.1","port":30513,"priority":0,"weight":1},{"host":"127.0.0.1","port":30514,"priority":0,"weight":1}] +--- ignore_error_log From 3f7fbaa41a0803475afce972a1fb4808ec06e29d Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Tue, 29 Nov 2022 13:58:07 +0800 Subject: [PATCH 20/27] feat: fix ci error --- t/discovery/consul_dump.t | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 2e01b638f721..2e8a7c1e2303 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -54,7 +54,9 @@ location /v1/agent { --- request eval [ "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_a2", "PUT /v1/agent/service/deregister/service_b1", + "PUT /v1/agent/service/deregister/service_b2", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":8002,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] @@ -64,6 +66,8 @@ location /v1/agent { '', '', '', + '', + '', ] From 5cdd92e07aaf429eab38ceab39b585cf5ce223e6 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 30 Nov 2022 03:46:29 +0000 Subject: [PATCH 21/27] feat: fix code review --- apisix/discovery/consul/init.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index b16510da5f6d..f3f351c8fd32 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -123,7 +123,7 @@ local function update_all_services(server_name_prefix, data) end consul_services[server_name_prefix] = up_services - log.info("update all services: ", core.json.encode(all_services)) + log.info("update all services: ", json_delay_encode(all_services, true)) end @@ -233,7 +233,7 @@ function _M.connect(premature, consul_server, retry_delay) ", watch_result status: ", watch_result.status, ", watch_result.headers.index: ", watch_result.headers['X-Consul-Index'], ", consul_server.index: ", consul_server.index, - ", consul_server: ", json_delay_encode(consul_server)) + ", consul_server: ", json_delay_encode(consul_server, true)) -- if current index different last index then update service if consul_server.index ~= watch_result.headers['X-Consul-Index'] then @@ -270,8 +270,8 @@ function _M.connect(premature, consul_server, retry_delay) -- decode body, decode json, update service, error handling if result.body then log.notice("server_name: ", consul_server.consul_server_url, - ", header: ", core.json.encode(result.headers, true), - ", body: ", core.json.encode(result.body, true)) + ", header: ", json_delay_encode(result.headers, true), + ", body: ", json_delay_encode(result.body, true)) update_all_services(consul_server.consul_server_url, result.body) --update events local ok, err = events.post(events_list._source, events_list.updating, all_services) @@ -362,7 +362,7 @@ function _M.init_worker() return end - log.notice("consul_conf: ", core.json.encode(consul_conf)) + log.notice("consul_conf: ", json_delay_encode(consul_conf, true)) default_weight = consul_conf.weight -- set default service, used when the server node cannot be found if consul_conf.default_service then @@ -381,7 +381,7 @@ function _M.init_worker() error(err) return end - log.info("consul_server_list: ", core.json.encode(consul_servers_list)) + log.info("consul_server_list: ", json_delay_encode(consul_servers_list, true)) consul_services = core.table.new(0, 1) -- success or failure From bff5d395478da6d080eddc91ad1a927348606ece Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 30 Nov 2022 16:15:30 +0800 Subject: [PATCH 22/27] feat: fix code review --- Makefile | 2 +- apisix/discovery/consul/init.lua | 28 ++++++++++++++-------------- conf/config-default.yaml | 2 +- docs/en/latest/discovery/consul.md | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 41d070ac1c49..9372d25258bc 100644 --- a/Makefile +++ b/Makefile @@ -284,8 +284,8 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery $(ENV_INSTALL) apisix/discovery/*.lua $(ENV_INST_LUADIR)/apisix/discovery/ $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul_kv,consul,dns,eureka,nacos,kubernetes,tars} - $(ENV_INSTALL) apisix/discovery/consul_kv/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul_kv $(ENV_INSTALL) apisix/discovery/consul/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul + $(ENV_INSTALL) apisix/discovery/consul_kv/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul_kv $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns $(ENV_INSTALL) apisix/discovery/eureka/*.lua $(ENV_INST_LUADIR)/apisix/discovery/eureka $(ENV_INSTALL) apisix/discovery/kubernetes/*.lua $(ENV_INST_LUADIR)/apisix/discovery/kubernetes diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index f3f351c8fd32..4d81e8da6166 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -141,7 +141,7 @@ local function read_dump_services() return end - local entity, err = core.json.decode(data) + local entity, err = core.json.decode(data) if not entity then log.error("decoded dump data got error: ", err, ", file content: ", data) return @@ -192,6 +192,16 @@ local function show_dump_file() return 200, data end +local function get_retry_delay(retry_delay) + if not retry_delay then + retry_delay = 1 + else + retry_delay = retry_delay * 4 + end + + return retry_delay +end + function _M.connect(premature, consul_server, retry_delay) if premature then @@ -217,12 +227,7 @@ function _M.connect(premature, consul_server, retry_delay) ", got watch result: ", json_delay_encode(watch_result, true), ", with error: ", watch_error_info) - if not retry_delay then - retry_delay = 1 - else - retry_delay = retry_delay * 4 - end - + retry_delay = get_retry_delay(retry_delay) log.warn("retry connecting consul after ", retry_delay, " seconds") core_sleep(retry_delay) @@ -250,12 +255,7 @@ function _M.connect(premature, consul_server, retry_delay) ", got result: ", json_delay_encode(result, true), ", with error: ", error_info) - if not retry_delay then - retry_delay = 1 - else - retry_delay = retry_delay * 4 - end - + retry_delay = get_retry_delay(retry_delay) log.warn("retry connecting consul after ", retry_delay, " seconds") core_sleep(retry_delay) @@ -335,7 +335,7 @@ local function format_consul_params(consul_conf) }) end - return consul_server_list + return consul_server_list, nil end diff --git a/conf/config-default.yaml b/conf/config-default.yaml index c1ccc6bfd77f..256a746bfe23 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -313,7 +313,7 @@ nginx_config: # config for render the template to generate n # servers: # - "http://127.0.0.1:8500" # - "http://127.0.0.1:8600" -# skip_services: # if you need to skip special keys +# skip_services: # if you need to skip special services # - "service_a" # timeout: # connect: 2000 # default 2000 ms diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md index 2bb12afc278e..5b26d3a0a79a 100644 --- a/docs/en/latest/discovery/consul.md +++ b/docs/en/latest/discovery/consul.md @@ -37,7 +37,7 @@ discovery: servers: - "http://127.0.0.1:8500" - "http://127.0.0.1:8600" # `http://127.0.0.1:8500` and `http://127.0.0.1:8600` are different clusters - skip_services: # if you need to skip special keys + skip_services: # if you need to skip special services - "service_a" timeout: connect: 1000 # default 2000 ms From 44cd5a4881ecd2923b727d0559970a4a45e16801 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 30 Nov 2022 16:43:48 +0800 Subject: [PATCH 23/27] feat: fix code review --- docs/en/latest/config.json | 2 +- docs/zh/latest/config.json | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index e7b5a5ba233c..364f08f29873 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -263,8 +263,8 @@ "items": [ "discovery", "discovery/dns", - "discovery/consul_kv", "discovery/consul", + "discovery/consul_kv", "discovery/nacos", "discovery/eureka", "discovery/control-plane-service-discovery", diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json index 2657f45ce55f..f3369c8b4cae 100644 --- a/docs/zh/latest/config.json +++ b/docs/zh/latest/config.json @@ -242,6 +242,8 @@ "items": [ "discovery", "discovery/dns", + "discovery/consul", + "discovery/consul_kv", "discovery/nacos", "discovery/eureka", "discovery/control-plane-service-discovery", From 99fd58a5f49c6ffef39983b8b227a412328eb548 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Wed, 30 Nov 2022 16:44:57 +0800 Subject: [PATCH 24/27] feat: fix code review --- docs/zh/latest/config.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json index f3369c8b4cae..2657f45ce55f 100644 --- a/docs/zh/latest/config.json +++ b/docs/zh/latest/config.json @@ -242,8 +242,6 @@ "items": [ "discovery", "discovery/dns", - "discovery/consul", - "discovery/consul_kv", "discovery/nacos", "discovery/eureka", "discovery/control-plane-service-discovery", From 2e5d391ac6412c5e1758ff6d7969fa03690662ca Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Thu, 1 Dec 2022 15:03:09 +0800 Subject: [PATCH 25/27] feat: fix code review --- apisix/discovery/consul/init.lua | 2 - docs/en/latest/discovery/consul.md | 2 +- t/discovery/consul.t | 55 +++++++++++++------------- t/discovery/consul_dump.t | 63 ++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 30 deletions(-) diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index 4d81e8da6166..7b05059c4340 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -379,7 +379,6 @@ function _M.init_worker() local consul_servers_list, err = format_consul_params(consul_conf) if err then error(err) - return end log.info("consul_server_list: ", json_delay_encode(consul_servers_list, true)) @@ -389,7 +388,6 @@ function _M.init_worker() local ok, err = ngx_timer_at(0, _M.connect, server) if not ok then error("create consul got error: " .. err) - return end if server.keepalive == false then diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md index 5b26d3a0a79a..17e13b00142e 100644 --- a/docs/en/latest/discovery/consul.md +++ b/docs/en/latest/discovery/consul.md @@ -34,7 +34,7 @@ First of all, we need to add following configuration in `conf/config.yaml` : ```yaml discovery: consul: - servers: + servers: # make sure service name is unique in these consul servers - "http://127.0.0.1:8500" - "http://127.0.0.1:8600" # `http://127.0.0.1:8500` and `http://127.0.0.1:8600` are different clusters skip_services: # if you need to skip special services diff --git a/t/discovery/consul.t b/t/discovery/consul.t index 8797f8912fc4..3c8b4d93503f 100644 --- a/t/discovery/consul.t +++ b/t/discovery/consul.t @@ -125,7 +125,7 @@ location /consul1 { } location /consul2 { - rewrite ^/consul2/(.*) /v1/catalog/services/$1 break; + rewrite ^/consul2/(.*) /v1/agent/service/$1 break; proxy_pass http://127.0.0.1:8600; } --- pipelined_requests eval @@ -134,13 +134,17 @@ location /consul2 { "PUT /consul1/deregister/service_b1", "PUT /consul1/deregister/service_a2", "PUT /consul1/deregister/service_b2", + "PUT /consul2/deregister/service_a1", + "PUT /consul2/deregister/service_b1", + "PUT /consul2/deregister/service_a2", + "PUT /consul2/deregister/service_b2", "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /consul1/register\n" . "{\"ID\":\"service_a2\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30512,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /consul1/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /consul1/register\n" . "{\"ID\":\"service_b2\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval -["", "", "", "", "", "", "", ""] +["", "", "", "", "", "", "", "", "", "", "", ""] @@ -371,7 +375,7 @@ location /sleep { -=== TEST 8: prepare healthy and unhealthy nodes +=== TEST 8: clean nodes --- config location /v1/agent { proxy_pass http://127.0.0.1:8500; @@ -380,38 +384,16 @@ location /v1/agent { [ "PUT /v1/agent/service/deregister/service_a1", "PUT /v1/agent/service/deregister/service_a2", - "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", - "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_a2\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.2\",\"Port\":1988,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval [ '', '', - '', - '', ] -=== TEST 9: clean nodes ---- config -location /v1/agent { - proxy_pass http://127.0.0.1:8500; -} ---- request eval -[ - "PUT /v1/agent/service/deregister/service_a1", - "PUT /v1/agent/service/deregister/service_a2", -] ---- response_body eval -[ - '', - '', -] - - - -=== TEST 10: test consul short connect type +=== TEST 9: test consul short connect type --- yaml_config apisix: node_listen: 1984 @@ -469,7 +451,7 @@ location /sleep { -=== TEST 11: retry when Consul can't be reached (long connect type) +=== TEST 10: retry when Consul can't be reached (long connect type) --- yaml_config apisix: node_listen: 1984 @@ -518,6 +500,25 @@ retry connecting consul after 4 seconds +=== TEST 11: prepare healthy and unhealthy nodes +--- config +location /v1/agent { + proxy_pass http://127.0.0.1:8500; +} +--- request eval +[ + "PUT /v1/agent/service/deregister/service_a1", + "PUT /v1/agent/service/deregister/service_a2", + "PUT /v1/agent/service/deregister/service_b1", + "PUT /v1/agent/service/deregister/service_b2", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b2\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", +] +--- response_body eval +['', '', '', '', '', ''] + + + === TEST 12: test health checker --- yaml_config eval: $::yaml_config --- apisix_yaml diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 2e8a7c1e2303..459a308deec4 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -397,3 +397,66 @@ GET /v1/discovery/consul/show_dump_file --- error_code: 503 --- error_log connect consul + + + +=== TEST 14: prepare nodes with different consul clusters +--- config +location /consul1 { + rewrite ^/consul1/(.*) /v1/agent/service/$1 break; + proxy_pass http://127.0.0.1:8500; +} + +location /consul2 { + rewrite ^/consul2/(.*) /v1/agent/service/$1 break; + proxy_pass http://127.0.0.1:8600; +} +--- pipelined_requests eval +[ + "PUT /consul1/deregister/service_a1", + "PUT /consul1/deregister/service_b1", + "PUT /consul1/deregister/service_a2", + "PUT /consul1/deregister/service_b2", + "PUT /consul2/deregister/service_a1", + "PUT /consul2/deregister/service_b1", + "PUT /consul2/deregister/service_a2", + "PUT /consul2/deregister/service_b2", + "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", + "PUT /consul2/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30517,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", +] +--- response_body eval +["", "", "", "", "", "", "", "", "", ""] + + + +=== TEST 15: show dump services with different consul clusters +--- yaml_config +apisix: + node_listen: 1984 + enable_control: true +discovery: + consul: + servers: + - "http://127.0.0.1:8500" + - "http://127.0.0.1:8600" + dump: + path: "consul.dump" + load_on_init: false +--- config + location /bonjour { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + ngx.sleep(2) + + local code, body, res = t.test('/v1/discovery/consul/show_dump_file', + ngx.HTTP_GET) + local entity = json.decode(res) + ngx.say(json.encode(entity.services)) + } + } +--- timeout: 3 +--- request +GET /bonjour +--- response_body +{"service_a":[{"host":"127.0.0.1","port":30511,"weight":1}],"service_b":[{"host":"127.0.0.1","port":30517,"weight":1}]} From 46bf152138befccec8c18a02aaa692618a2aeb03 Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Mon, 5 Dec 2022 13:56:15 +0800 Subject: [PATCH 26/27] feat: fix code review --- Makefile | 2 +- apisix/discovery/consul/init.lua | 12 ++++++------ t/discovery/consul.t | 15 ++++++--------- t/discovery/consul_dump.t | 23 +++++++---------------- 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 488b0a0a03a8..a9602194f2a0 100644 --- a/Makefile +++ b/Makefile @@ -283,7 +283,7 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery $(ENV_INSTALL) apisix/discovery/*.lua $(ENV_INST_LUADIR)/apisix/discovery/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul_kv,consul,dns,eureka,nacos,kubernetes,tars} + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul,consul_kv,dns,eureka,nacos,kubernetes,tars} $(ENV_INSTALL) apisix/discovery/consul/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul $(ENV_INSTALL) apisix/discovery/consul_kv/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul_kv $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua index 7b05059c4340..dd8275e7d857 100644 --- a/apisix/discovery/consul/init.lua +++ b/apisix/discovery/consul/init.lua @@ -31,6 +31,7 @@ local ngx_timer_at = ngx.timer.at local ngx_timer_every = ngx.timer.every local log = core.log local json_delay_encode = core.json.delay_encode +local ngx_worker_id = ngx.worker.id local all_services = core.table.new(0, 5) local default_service @@ -73,7 +74,7 @@ function _M.nodes(service_name) return default_service and {default_service} end - log.info("process id: ", ngx.worker.id(), ", all_services[", service_name, "] = ", + log.info("process id: ", ngx_worker_id(), ", all_services[", service_name, "] = ", json_delay_encode(resp_list, true)) return resp_list @@ -86,22 +87,21 @@ local function parse_instance(node) if service_name and skip_service_map[service_name] then return false end + -- "" means metadata of the service return true, host, tonumber(port), "", service_name end local function update_all_services(server_name_prefix, data) - local sn local up_services = core.table.new(0, #data) local weight = default_weight for _, node in pairs(data) do local succ, ip, port, metadata, server_name = parse_instance(node) if succ then - sn = server_name - local nodes = up_services[sn] + local nodes = up_services[server_name] if not nodes then nodes = core.table.new(1, 0) - up_services[sn] = nodes + up_services[server_name] = nodes end core.table.insert(nodes, { host = ip, @@ -357,7 +357,7 @@ function _M.init_worker() "updating" ) - if 0 ~= ngx.worker.id() then + if 0 ~= ngx_worker_id() then events.register(discovery_consul_callback, events_list._source, events_list.updating) return end diff --git a/t/discovery/consul.t b/t/discovery/consul.t index 3c8b4d93503f..39c5ab287b50 100644 --- a/t/discovery/consul.t +++ b/t/discovery/consul.t @@ -143,8 +143,8 @@ location /consul2 { "PUT /consul1/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /consul1/register\n" . "{\"ID\":\"service_b2\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] ---- response_body eval -["", "", "", "", "", "", "", "", "", "", "", ""] +--- error_code eval +[200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200] @@ -385,11 +385,8 @@ location /v1/agent { "PUT /v1/agent/service/deregister/service_a1", "PUT /v1/agent/service/deregister/service_a2", ] ---- response_body eval -[ - '', - '', -] +--- error_code eval +[200, 200] @@ -514,8 +511,8 @@ location /v1/agent { "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30513,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b2\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30514,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] ---- response_body eval -['', '', '', '', '', ''] +--- error_code eval +[200, 200, 200, 200, 200, 200] diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 459a308deec4..366c76a989fb 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -61,14 +61,8 @@ location /v1/agent { "PUT /v1/agent/service/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":8002,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] --- response_body eval -[ - '', - '', - '', - '', - '', - '', -] +--- error_code eval +[200, 200, 200, 200, 200, 200] @@ -83,7 +77,7 @@ discovery: - "http://127.0.0.1:8500" dump: path: "consul.dump" - load_on_init: true + load_on_init: false --- config location /t { content_by_lua_block { @@ -148,11 +142,8 @@ location /v1/agent { "PUT /v1/agent/service/deregister/service_a1", "PUT /v1/agent/service/deregister/service_b1", ] ---- response_body eval -[ - '', - '' -] +--- error_code eval +[200, 200] @@ -424,8 +415,8 @@ location /consul2 { "PUT /consul1/register\n" . "{\"ID\":\"service_a1\",\"Name\":\"service_a\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30511,\"Meta\":{\"service_a_version\":\"4.0\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", "PUT /consul2/register\n" . "{\"ID\":\"service_b1\",\"Name\":\"service_b\",\"Tags\":[\"primary\",\"v1\"],\"Address\":\"127.0.0.1\",\"Port\":30517,\"Meta\":{\"service_b_version\":\"4.1\"},\"EnableTagOverride\":false,\"Weights\":{\"Passing\":10,\"Warning\":1}}", ] ---- response_body eval -["", "", "", "", "", "", "", "", "", ""] +--- error_code eval +[200, 200, 200, 200, 200, 200, 200, 200, 200, 200] From 115af5b99242d671b301e061607e8c9e13b4571b Mon Sep 17 00:00:00 2001 From: Fabriceli Date: Mon, 5 Dec 2022 13:57:17 +0800 Subject: [PATCH 27/27] feat: fix code review --- t/discovery/consul_dump.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/discovery/consul_dump.t b/t/discovery/consul_dump.t index 366c76a989fb..b229fb7a4666 100644 --- a/t/discovery/consul_dump.t +++ b/t/discovery/consul_dump.t @@ -77,7 +77,7 @@ discovery: - "http://127.0.0.1:8500" dump: path: "consul.dump" - load_on_init: false + load_on_init: true --- config location /t { content_by_lua_block {