Skip to content

Commit

Permalink
feat: support reading configuration form xds(mvp) (#6614)
Browse files Browse the repository at this point in the history
  • Loading branch information
tzssangglass authored and spacewander committed Jun 30, 2022
1 parent d5bfbae commit 555140a
Show file tree
Hide file tree
Showing 9 changed files with 338 additions and 5 deletions.
7 changes: 6 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
test_dir:
- t/plugin
- t/admin t/cli t/config-center-yaml t/control t/core t/debug t/discovery t/error_page t/misc
- t/node t/router t/script t/stream-node t/utils t/wasm
- t/node t/router t/script t/stream-node t/utils t/wasm t/xds-library

runs-on: ${{ matrix.platform }}
timeout-minutes: 90
Expand Down Expand Up @@ -90,6 +90,11 @@ jobs:
sudo dpkg -i tinygo_${TINYGO_VER}_amd64.deb
cd t/wasm && find . -type f -name "*.go" | xargs -Ip tinygo build -o p.wasm -scheduler=none -target=wasi p
- name: Build xDS library
run: |
cd t/xds-library
go build -o libxds.so -buildmode=c-shared main.go
- name: Linux Before install
run: sudo ./ci/${{ matrix.os_name }}_runner.sh before_install

Expand Down
4 changes: 4 additions & 0 deletions apisix/cli/ngx_tpl.lua
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,10 @@ http {
lua_shared_dict ext-plugin {* http.lua_shared_dict["ext-plugin"] *}; # cache for ext-plugin
{% end %}
{% if config_center == "xds" then %}
lua_shared_dict xds-route-config 10m;
{% end %}
# for custom shared dict
{% if http.custom_lua_shared_dict then %}
{% for cache_key, cache_size in pairs(http.custom_lua_shared_dict) do %}
Expand Down
2 changes: 1 addition & 1 deletion apisix/cli/schema.lua
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ local config_schema = {
apisix = {
properties = {
config_center = {
enum = {"etcd", "yaml"},
enum = {"etcd", "yaml", "xds"},
},
lua_module_hook = {
pattern = "^[a-zA-Z._-]+$",
Expand Down
120 changes: 120 additions & 0 deletions apisix/core/config_xds.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
--
-- Licensed to the Apache Software Foundation (ASF) under one or more
-- contributor license agreements. See the NOTICE file distributed with
-- this work for additional information regarding copyright ownership.
-- The ASF licenses this file to You under the Apache License, Version 2.0
-- (the "License"); you may not use this file except in compliance with
-- the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

--- Get configuration form ngx.shared.DICT.
--
-- @module core.config_xds

local base = require("resty.core.base")
local config_local = require("apisix.core.config_local")
local table = table
local error = error
local is_http = ngx.config.subsystem == "http"
local io = io
local io_open = io.open
local io_close = io.close
local package = package
local new_tab = base.new_tab
local ffi = require ("ffi")
local C = ffi.C
local route_config = ngx.shared["xds-route-config"]
local ngx_re_match = ngx.re.match
local ngx_re_gmatch = ngx.re.gmatch

local xds_lib_name = "libxds.so"


local process
if is_http then
process = require("ngx.process")
end


ffi.cdef[[
extern void initial(void* route_zone_ptr);
]]


local _M = {
version = 0.1,
local_conf = config_local.local_conf,
}


-- todo: refactor this function in chash.lua and radixtree.lua
local function load_shared_lib(lib_name)
local cpath = package.cpath
local tried_paths = new_tab(32, 0)
local i = 1

local iter, err = ngx_re_gmatch(cpath, "[^;]+", "jo")
if not iter then
error("failed to gmatch: " .. err)
end

while true do
local it = iter()
local fpath
fpath, err = ngx_re_match(it[0], "(.*/)", "jo")
if err then
error("failed to match: " .. err)
end
local spath = fpath[0] .. lib_name

local f = io_open(spath)
if f ~= nil then
io_close(f)
return ffi.load(spath)
end
tried_paths[i] = spath
i = i + 1

if not it then
break
end
end

return nil, tried_paths
end


local function load_libxds(lib_name)
local xdsagent, tried_paths = load_shared_lib(lib_name)

if not xdsagent then
tried_paths[#tried_paths + 1] = 'tried above paths but can not load ' .. lib_name
error("can not load xds library, tried paths: " ..
table.concat(tried_paths, '\r\n', 1, #tried_paths))
end

local route_zone = C.ngx_http_lua_ffi_shdict_udata_to_zone(route_config[1])
local route_shd_cdata = ffi.cast("void*", route_zone)
xdsagent.initial(route_shd_cdata)
end



function _M.init_worker()
if process.type() == "privileged agent" then
load_libxds(xds_lib_name)
end

return true
end


return _M
2 changes: 2 additions & 0 deletions apisix/core/config_yaml.lua
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,8 @@ end
function _M.init_worker()
-- sync data in each non-master process
ngx.timer.every(1, read_apisix_yaml)

return true
end


Expand Down
8 changes: 6 additions & 2 deletions apisix/init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,12 @@ function _M.http_init_worker()
plugin_config.init_worker()
require("apisix.consumer").init_worker()

if core.config == require("apisix.core.config_yaml") then
core.config.init_worker()
if core.config.init_worker then
local ok, err = core.config.init_worker()
if not ok then
core.log.error("failed to init worker process of ", core.config.type,
" config center, err: ", err)
end
end

apisix_upstream.init_worker()
Expand Down
3 changes: 2 additions & 1 deletion t/APISIX.pm
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ apisix:
_EOC_
}

my $lua_deps_path = <<_EOC_;
my $lua_deps_path = $block->lua_deps_path // <<_EOC_;
lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;";
lua_package_cpath "$apisix_home/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;";
_EOC_
Expand Down Expand Up @@ -509,6 +509,7 @@ _EOC_
lua_shared_dict ext-plugin 1m;
lua_shared_dict kubernetes 1m;
lua_shared_dict tars 1m;
lua_shared_dict xds-route-config 1m;
proxy_ssl_name \$upstream_host;
proxy_ssl_server_name on;
Expand Down
105 changes: 105 additions & 0 deletions t/xds-library/config_xds.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
use t::APISIX 'no_plan';

use Cwd qw(cwd);
my $apisix_home = $ENV{APISIX_HOME} || cwd();

repeat_each(1);
no_long_string();
no_root_location();
log_level("info");

add_block_preprocessor(sub {
my ($block) = @_;

if (!$block->request) {
$block->set_value("request", "GET /t");
}

if (!$block->no_error_log) {
$block->set_value("no_error_log", "[error]\n[alert]");
}

my $lua_deps_path = $block->lua_deps_path // <<_EOC_;
lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;";
lua_package_cpath "$apisix_home/?.so;$apisix_home/t/xds-library/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;";
_EOC_

$block->set_value("lua_deps_path", $lua_deps_path);

my $extra_init_by_lua = <<_EOC_;
--
local config_xds = require("apisix.core.config_xds")
local inject = function(mod, name)
local old_f = mod[name]
mod[name] = function (...)
ngx.log(ngx.WARN, "config_xds run ", name)
return { true }
end
end
inject(config_xds, "new")
_EOC_

$block->set_value("extra_init_by_lua", $extra_init_by_lua);

if (!$block->yaml_config) {
my $yaml_config = <<_EOC_;
apisix:
node_listen: 1984
config_center: xds
enable_admin: false
_EOC_

$block->set_value("yaml_config", $yaml_config);
}
});

run_tests;

__DATA__
=== TEST 1: load xDS library successfully
--- config
location /t {
content_by_lua_block {
ngx.say("ok")
}
}
--- no_error_log eval
qr/can not load xDS library/
=== TEST 2: read data form shdict that wirted by xDS library
--- config
location /t {
content_by_lua_block {
-- wait for xds library sync data
ngx.sleep(1.5)
local core = require("apisix.core")
local value = ngx.shared["xds-route-config"]:get("/apisix/routes/1")
local route_conf, err = core.json.decode(value)
local json_encode = require("toolkit.json").encode
ngx.say(json_encode(route_conf))
}
}
--- response_body
{"create_time":1646972532,"id":"1","priority":0,"status":1,"update_time":1647250524,"upstream":{"hash_on":"vars","nodes":[{"host":"127.0.0.1","port":80,"priority":0,"weight":1}],"pass_host":"pass","scheme":"http","type":"roundrobin"},"uri":"/hello"}
Loading

0 comments on commit 555140a

Please sign in to comment.