diff --git a/eval/policy/admission-control/attach1.toml b/eval/policy/admission-control/attach1.toml new file mode 100644 index 00000000..460dc5c6 --- /dev/null +++ b/eval/policy/admission-control/attach1.toml @@ -0,0 +1,33 @@ +addon_engine = "HelloAclSenderEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "HelloAclSenderEngine", + 0, + 0, + ], + [ + "HelloAclSenderEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "HelloAclSenderEngine", + 0, + 0, + ], + [ + "HelloAclSenderEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/admission-control/attach2.toml b/eval/policy/admission-control/attach2.toml new file mode 100644 index 00000000..fe3147f4 --- /dev/null +++ b/eval/policy/admission-control/attach2.toml @@ -0,0 +1,33 @@ +addon_engine = "AdmissionControlEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "AdmissionControlEngine", + 0, + 0, + ], + [ + "AdmissionControlEngine", + "HelloAclSenderEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "HelloAclSenderEngine", + "AdmissionControlEngine", + 0, + 0, + ], + [ + "AdmissionControlEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "HelloAclSenderEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/admission-control/detach.toml b/eval/policy/admission-control/detach.toml new file mode 100644 index 00000000..48793f78 --- /dev/null +++ b/eval/policy/admission-control/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "HelloAclSenderEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [] +op = "detach" diff --git a/eval/policy/fault-server/attach.toml b/eval/policy/fault-server/attach.toml new file mode 100644 index 00000000..4dcca7ff --- /dev/null +++ b/eval/policy/fault-server/attach.toml @@ -0,0 +1,33 @@ +addon_engine = "FaultServerEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "FaultServerEngine", + 0, + 0, + ], + [ + "FaultServerEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "FaultServerEngine", + 0, + 0, + ], + [ + "FaultServerEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/fault-server/collect.py b/eval/policy/fault-server/collect.py new file mode 100755 index 00000000..009eec0d --- /dev/null +++ b/eval/policy/fault-server/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Fault') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Fault') + + +for f in glob.glob(OD+"/policy/fault/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'ADN+mRPC', f) diff --git a/eval/policy/fault-server/config.toml b/eval/policy/fault-server/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/fault-server/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/fault-server/detach.toml b/eval/policy/fault-server/detach.toml new file mode 100644 index 00000000..a6b477c8 --- /dev/null +++ b/eval/policy/fault-server/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "FaultServerEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/fault-server/rpc_bench_tput_32b.toml b/eval/policy/fault-server/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..d1d46cd5 --- /dev/null +++ b/eval/policy/fault-server/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/fault/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "fault" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/fault-server/start_traffic.sh b/eval/policy/fault-server/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/fault-server/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/hello-acl-receiver/detach.toml b/eval/policy/hello-acl-receiver/detach.toml index d77a7d9a..23f93a0e 100644 --- a/eval/policy/hello-acl-receiver/detach.toml +++ b/eval/policy/hello-acl-receiver/detach.toml @@ -1,4 +1,4 @@ addon_engine = "HelloAclReceiverEngine" tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] -rx_channels_replacements = [] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] op = "detach" diff --git a/eval/policy/logging-server/README.md b/eval/policy/logging-server/README.md new file mode 100644 index 00000000..2e14d4dc --- /dev/null +++ b/eval/policy/logging-server/README.md @@ -0,0 +1,4 @@ +If you want to measure latency + +change `--concurrency 128` to `--concurrency 1` and append +`--log-latency` to the end of the args in `rpc_bench_tput_32b.toml`. diff --git a/eval/policy/logging-server/attach.toml b/eval/policy/logging-server/attach.toml new file mode 100644 index 00000000..363a235a --- /dev/null +++ b/eval/policy/logging-server/attach.toml @@ -0,0 +1,31 @@ +addon_engine = "LoggingServerEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "LoggingServerEngine", + 0, + 0, + ], + [ + "LoggingServerEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "LoggingServerEngine", + 0, + 0, + ], + [ + "LoggingServerEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" diff --git a/eval/policy/logging-server/collect.py b/eval/policy/logging-server/collect.py new file mode 100755 index 00000000..5aeb37f3 --- /dev/null +++ b/eval/policy/logging-server/collect.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Logging') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Logging') + + +for f in glob.glob(OD+"/policy/logging/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'Native mRPC', f) diff --git a/eval/policy/logging-server/config.toml b/eval/policy/logging-server/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/logging-server/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/logging-server/detach.toml b/eval/policy/logging-server/detach.toml new file mode 100644 index 00000000..a236b2c2 --- /dev/null +++ b/eval/policy/logging-server/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "LoggingServerEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/logging-server/rpc_bench_tput_32b.toml b/eval/policy/logging-server/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..b8aae34e --- /dev/null +++ b/eval/policy/logging-server/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/logging/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "logging" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l info" +dependencies = [0] diff --git a/eval/policy/logging-server/run_policy.py b/eval/policy/logging-server/run_policy.py new file mode 100755 index 00000000..02baaf74 --- /dev/null +++ b/eval/policy/logging-server/run_policy.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import json +import subprocess +import pathlib +import os +from os.path import dirname +import time +import toml +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + +SCRIPTDIR = pathlib.Path(__file__).parent.resolve() +CONFIG_PATH = os.path.join(SCRIPTDIR, "config.toml") + +config = toml.load(CONFIG_PATH) +workdir = config["workdir"] +workdir = dirname(dirname(os.path.expanduser(workdir))) +env = {**os.environ, **config['env']} + +os.chdir(workdir) +os.makedirs(f"{OD}/policy/null", exist_ok=True) + +cmd = f'''cargo run --release -p benchmark --bin launcher -- -o {OD} --timeout=120 +--benchmark {os.path.join(SCRIPTDIR, 'rpc_bench_tput_32b.toml')} +--configfile { os.path.join(SCRIPTDIR, 'config.toml')}''' +workload = subprocess.Popen(cmd.split()) +time.sleep(30) + +list_cmd = f"cargo run --release --bin list -- --dump {OD}/policy/list.json" +subprocess.run(list_cmd.split(), env=env) + +with open(f"{OD}/policy/list.json", "r") as fin: + content = fin.read() + print(content) + data = json.loads(content) +mrpc_pid = None +mrpc_sid = None +for subscription in data: + pid = subscription["pid"] + sid = subscription["sid"] + engines = [x[1] for x in subscription["engines"]] + if "MrpcEngine" in engines: + mrpc_pid = pid + mrpc_sid = sid + +print("Start to attach policy") +attach_config = os.path.join(SCRIPTDIR, "attach.toml") +attach_cmd = f"cargo run --release --bin addonctl -- --config {attach_config} --pid {mrpc_pid} --sid {mrpc_sid}" +subprocess.run(attach_cmd.split(), env=env) + +subprocess.run(list_cmd.split(), env=env) +with open(f"{OD}/policy/list.json", "r") as fin: + print(fin.read()) + +workload.wait() diff --git a/eval/policy/metrics-server/attach.toml b/eval/policy/metrics-server/attach.toml new file mode 100644 index 00000000..7c9d0caa --- /dev/null +++ b/eval/policy/metrics-server/attach.toml @@ -0,0 +1,33 @@ +addon_engine = "MetricsServerEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "MetricsServerEngine", + 0, + 0, + ], + [ + "MetricsServerEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "MetricsServerEngine", + 0, + 0, + ], + [ + "MetricsServerEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +''' diff --git a/eval/policy/metrics-server/collect.py b/eval/policy/metrics-server/collect.py new file mode 100755 index 00000000..009eec0d --- /dev/null +++ b/eval/policy/metrics-server/collect.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +from typing import List +import glob +import sys + +OD = "/tmp/mrpc-eval" +if len(sys.argv) >= 2: + OD = sys.argv[1] + + +def convert_msg_size(s: str) -> int: + if s.endswith('gb'): + return int(s[:-2]) * 1024 * 1024 * 1024 + if s.endswith('mb'): + return int(s[:-2]) * 1024 * 1024 + if s.endswith('kb'): + return int(s[:-2]) * 1024 + if s.endswith('b'): + return int(s[:-1]) + + raise ValueError(f"unknown input: {s}") + + +def get_rate(path: str) -> List[float]: + rates = [] + with open(path, 'r') as fin: + for line in fin: + words = line.strip().split(' ') + if words[-3] == 'rps,': + rate = float(words[-4]) + rates.append(rate) + return rates[1:] + + +def load_result(sol_before, sol_after, f: str): + # print(f) + rates = get_rate(f) + before = rates[5:25] + after = rates[-25:-5] + for r in before: + print(f'{round(r/1000,2)},{sol_before},w/o Fault') + for r in after: + print(f'{round(r/1000,2)},{sol_after},w/ Fault') + + +for f in glob.glob(OD+"/policy/fault/rpc_bench_tput_32b/rpc_bench_client_danyang-04.stdout"): + load_result('mRPC', 'ADN+mRPC', f) diff --git a/eval/policy/metrics-server/config.toml b/eval/policy/metrics-server/config.toml new file mode 100644 index 00000000..d6ed4a5b --- /dev/null +++ b/eval/policy/metrics-server/config.toml @@ -0,0 +1,9 @@ +workdir = "~/nfs/Developing/livingshade/phoenix/experimental/mrpc" + +[env] +RUST_BACKTRACE = "1" +RUST_LOG_STYLE = "never" +CARGO_TERM_COLOR = "never" +PHOENIX_LOG = "info" +PROTOC = "/usr/bin/protoc" +PHOENIX_PREFIX = "/tmp/phoenix" diff --git a/eval/policy/metrics-server/detach.toml b/eval/policy/metrics-server/detach.toml new file mode 100644 index 00000000..88214f94 --- /dev/null +++ b/eval/policy/metrics-server/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "MetricsServerEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/metrics-server/rpc_bench_tput_32b.toml b/eval/policy/metrics-server/rpc_bench_tput_32b.toml new file mode 100644 index 00000000..d1d46cd5 --- /dev/null +++ b/eval/policy/metrics-server/rpc_bench_tput_32b.toml @@ -0,0 +1,15 @@ +name = "policy/fault/rpc_bench_tput_32b" +description = "Run rpc_bench benchmark" +group = "fault" +timeout_secs = 70 + +[[worker]] +host = "danyang-06" +bin = "rpc_bench_server" +args = "--port 5002 -l info --transport tcp" + +[[worker]] +host = "danyang-04" +bin = "rpc_bench_client" +args = "--transport tcp -c rdma0.danyang-06 --concurrency 128 --req-size 32 --duration 65 -i 1 --port 5002 -l error" +dependencies = [0] diff --git a/eval/policy/metrics-server/start_traffic.sh b/eval/policy/metrics-server/start_traffic.sh new file mode 100755 index 00000000..f0c5c3d1 --- /dev/null +++ b/eval/policy/metrics-server/start_traffic.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM SIGHUP EXIT + +OD=/tmp/mrpc-eval +if [[ $# -ge 1 ]]; then + OD=$1 +fi + +WORKDIR=$(dirname $(realpath $0)) +cd $WORKDIR + +# concurrency = 128 +cargo rr --bin launcher -- --output-dir ${OD} --timeout=120 --benchmark ./rpc_bench_tput_32b.toml --configfile ./config.toml & + +sleep 30 + +LIST_OUTPUT="${OD}"/policy/list.json +cargo rr --bin list -- --dump "${LIST_OUTPUT}" # Need to specifiy PHOENIX_PREFIX +cat "${LIST_OUTPUT}" +ARG_PID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .pid') +ARG_SID=$(cat "${LIST_OUTPUT}" | jq '.[] | select(.service == "Mrpc") | .sid') +echo $ARG_SID + +sleep 1 + +cargo run --bin addonctl -- --config ./attach.toml --pid ${ARG_PID} --sid ${ARG_SID} # Need to specifiy PHOENIX_PREFIX + +wait diff --git a/eval/policy/mutation-server/attach.toml b/eval/policy/mutation-server/attach.toml new file mode 100644 index 00000000..c3ded6ef --- /dev/null +++ b/eval/policy/mutation-server/attach.toml @@ -0,0 +1,31 @@ +addon_engine = "MutationServerEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "MutationServerEngine", + 0, + 0, + ], + [ + "MutationServerEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "MutationServerEngine", + 0, + 0, + ], + [ + "MutationServerEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" diff --git a/eval/policy/mutation-server/detach.toml b/eval/policy/mutation-server/detach.toml new file mode 100644 index 00000000..c92148aa --- /dev/null +++ b/eval/policy/mutation-server/detach.toml @@ -0,0 +1,6 @@ +addon_engine = "MutationServerEngine" +tx_channels_replacements = [ + ["MrpcEngine", "RpcAdapterEngine", 0, 0], +] +rx_channels_replacements = [] +op = "detach" diff --git a/eval/policy/mutation/attach.toml b/eval/policy/mutation/attach.toml new file mode 100644 index 00000000..b739a2e2 --- /dev/null +++ b/eval/policy/mutation/attach.toml @@ -0,0 +1,31 @@ +addon_engine = "MutationEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "MutationEngine", + 0, + 0, + ], + [ + "MutationEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "MutationEngine", + 0, + 0, + ], + [ + "MutationEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" diff --git a/eval/policy/mutation/detach.toml b/eval/policy/mutation/detach.toml new file mode 100644 index 00000000..465cc3e0 --- /dev/null +++ b/eval/policy/mutation/detach.toml @@ -0,0 +1,6 @@ +addon_engine = "LoggingEngine" +tx_channels_replacements = [ + ["MrpcEngine", "RpcAdapterEngine", 0, 0], +] +rx_channels_replacements = [] +op = "detach" diff --git a/eval/policy/ratelimit-drop-server/attach.toml b/eval/policy/ratelimit-drop-server/attach.toml new file mode 100644 index 00000000..49d9c82c --- /dev/null +++ b/eval/policy/ratelimit-drop-server/attach.toml @@ -0,0 +1,35 @@ +addon_engine = "RateLimitDropServerEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "RateLimitDropServerEngine", + 0, + 0, + ], + [ + "RateLimitDropServerEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "RateLimitDropServerEngine", + 0, + 0, + ], + [ + "RateLimitDropServerEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +requests_per_sec = 1 +bucket_size = 2 +''' diff --git a/eval/policy/ratelimit-drop-server/detach.toml b/eval/policy/ratelimit-drop-server/detach.toml new file mode 100644 index 00000000..9c45c4f0 --- /dev/null +++ b/eval/policy/ratelimit-drop-server/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "RateLimitDropServerEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/eval/policy/ratelimit-drop/attach.toml b/eval/policy/ratelimit-drop/attach.toml new file mode 100644 index 00000000..3aae53ac --- /dev/null +++ b/eval/policy/ratelimit-drop/attach.toml @@ -0,0 +1,35 @@ +addon_engine = "RateLimitDropEngine" +tx_channels_replacements = [ + [ + "MrpcEngine", + "RateLimitDropEngine", + 0, + 0, + ], + [ + "RateLimitDropEngine", + "TcpRpcAdapterEngine", + 0, + 0, + ], +] +rx_channels_replacements = [ + [ + "TcpRpcAdapterEngine", + "RateLimitDropEngine", + 0, + 0, + ], + [ + "RateLimitDropEngine", + "MrpcEngine", + 0, + 0, + ], +] +group = ["MrpcEngine", "TcpRpcAdapterEngine"] +op = "attach" +config_string = ''' +requests_per_sec = 2 +bucket_size = 10 +''' diff --git a/eval/policy/ratelimit-drop/detach.toml b/eval/policy/ratelimit-drop/detach.toml new file mode 100644 index 00000000..fcd8f868 --- /dev/null +++ b/eval/policy/ratelimit-drop/detach.toml @@ -0,0 +1,4 @@ +addon_engine = "RateLimitDropEngine" +tx_channels_replacements = [["MrpcEngine", "TcpRpcAdapterEngine", 0, 0]] +rx_channels_replacements = [["TcpRpcAdapterEngine", "MrpcEngine", 0, 0]] +op = "detach" diff --git a/experimental/mrpc/.gitignore b/experimental/mrpc/.gitignore index 2f7896d1..c2458ffa 100644 --- a/experimental/mrpc/.gitignore +++ b/experimental/mrpc/.gitignore @@ -1 +1,2 @@ target/ +generated/* \ No newline at end of file diff --git a/experimental/mrpc/Cargo.lock b/experimental/mrpc/Cargo.lock index 49357298..f862caf6 100644 --- a/experimental/mrpc/Cargo.lock +++ b/experimental/mrpc/Cargo.lock @@ -1576,6 +1576,29 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phoenix-admission-control" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "fnv", + "futures", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-admission-control", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-api" version = "0.1.0" @@ -1623,6 +1646,14 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "phoenix-api-policy-admission-control" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + [[package]] name = "phoenix-api-policy-delay" version = "0.1.0" @@ -1643,6 +1674,16 @@ dependencies = [ "serde", ] +[[package]] +name = "phoenix-api-policy-fault-server" +version = "0.1.0" +dependencies = [ + "itertools", + "phoenix-api", + "rand 0.8.5", + "serde", +] + [[package]] name = "phoenix-api-policy-fault2" version = "0.1.0" @@ -1694,6 +1735,46 @@ dependencies = [ "serde", ] +[[package]] +name = "phoenix-api-policy-logging-server" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + +[[package]] +name = "phoenix-api-policy-metrics" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + +[[package]] +name = "phoenix-api-policy-metrics-server" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + +[[package]] +name = "phoenix-api-policy-mutation" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + +[[package]] +name = "phoenix-api-policy-mutation-server" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + [[package]] name = "phoenix-api-policy-nofile-logging" version = "0.1.0" @@ -1728,6 +1809,22 @@ dependencies = [ "serde", ] +[[package]] +name = "phoenix-api-policy-ratelimit-drop" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + +[[package]] +name = "phoenix-api-policy-ratelimit-drop-server" +version = "0.1.0" +dependencies = [ + "phoenix-api", + "serde", +] + [[package]] name = "phoenix-api-rpc-adapter" version = "0.1.0" @@ -1830,6 +1927,30 @@ dependencies = [ "toml", ] +[[package]] +name = "phoenix-fault-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-fault-server", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-fault2" version = "0.1.0" @@ -1999,6 +2120,74 @@ dependencies = [ "toml", ] +[[package]] +name = "phoenix-logging-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-logging-server", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-metrics" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "fnv", + "futures", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-metrics", + "phoenix_common", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-metrics-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "fnv", + "futures", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-metrics-server", + "phoenix_common", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-mrpc" version = "0.1.0" @@ -2063,6 +2252,50 @@ dependencies = [ "uuid", ] +[[package]] +name = "phoenix-mutation" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "fnv", + "futures", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-mutation", + "phoenix_common", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-mutation-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "fnv", + "futures", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-mutation-server", + "phoenix_common", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-nofile-logging" version = "0.1.0" @@ -2137,6 +2370,53 @@ dependencies = [ "toml", ] +[[package]] +name = "phoenix-ratelimit-drop" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-ratelimit-drop", + "phoenix_common", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + +[[package]] +name = "phoenix-ratelimit-drop-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "chrono", + "futures", + "itertools", + "minstant", + "mrpc-derive", + "mrpc-marshal", + "nix", + "phoenix-api", + "phoenix-api-policy-ratelimit-drop-server", + "phoenix_common", + "rand 0.8.5", + "serde", + "serde_json", + "shm", + "thiserror", + "toml", +] + [[package]] name = "phoenix-rpc-adapter" version = "0.1.0" diff --git a/experimental/mrpc/Cargo.toml b/experimental/mrpc/Cargo.toml index b7f39c6c..8c3d8d3f 100644 --- a/experimental/mrpc/Cargo.toml +++ b/experimental/mrpc/Cargo.toml @@ -3,136 +3,155 @@ name = "mrpc" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [features] timing = ["dep:minstant"] -[dependencies] -phoenix-api-mrpc.workspace = true - -phoenix-api = { workspace = true, features = ["mrpc"] } -ipc = { workspace = true, features = ["customer"] } -shm = { workspace = true, features = ["mrpc"] } -mmap.workspace = true -phoenix-syscalls.workspace = true -shmalloc.workspace = true - -minstant = { workspace = true, optional = true } -thiserror.workspace = true -uuid.workspace = true -libc.workspace = true -fnv.workspace = true -memfd.workspace = true -lazy_static.workspace = true -tracing.workspace = true -dashmap.workspace = true -async-executor.workspace = true -arrayvec.workspace = true -async-trait.workspace = true -log.workspace = true -futures.workspace = true -serde_json.workspace = true -libnuma.workspace = true -slab.workspace = true -spin.workspace = true +[dependencies.phoenix-api-mrpc] +workspace = true + +[dependencies.phoenix-api] +workspace = true +features = ["mrpc"] + +[dependencies.ipc] +workspace = true +features = ["customer"] + +[dependencies.shm] +workspace = true +features = ["mrpc"] + +[dependencies.mmap] +workspace = true + +[dependencies.phoenix-syscalls] +workspace = true + +[dependencies.shmalloc] +workspace = true + +[dependencies.minstant] +workspace = true +optional = true + +[dependencies.thiserror] +workspace = true + +[dependencies.uuid] +workspace = true + +[dependencies.libc] +workspace = true + +[dependencies.fnv] +workspace = true + +[dependencies.memfd] +workspace = true + +[dependencies.lazy_static] +workspace = true + +[dependencies.tracing] +workspace = true + +[dependencies.dashmap] +workspace = true + +[dependencies.async-executor] +workspace = true + +[dependencies.arrayvec] +workspace = true + +[dependencies.async-trait] +workspace = true + +[dependencies.log] +workspace = true + +[dependencies.futures] +workspace = true + +[dependencies.serde_json] +workspace = true + +[dependencies.libnuma] +workspace = true + +[dependencies.slab] +workspace = true + +[dependencies.spin] +workspace = true [workspace] members = [ - # mrpc user-land libraries - "mrpc-build", - "mrpc-derive", - "mrpc-marshal", - # extension to phoenix-api - "phoenix-api/mrpc", - "phoenix-api/mrpclb", - "phoenix-api/rpc_adapter", - "phoenix-api/tcp_rpc_adapter", - "phoenix-api/load_balancer", - "phoenix-api/policy/null", - "phoenix-api/policy/ratelimit", - "phoenix-api/policy/qos", - "phoenix-api/policy/hotel-acl", - "phoenix-api/policy/logging", - "phoenix-api/policy/hello-acl-receiver", - "phoenix-api/policy/hello-acl-sender", - "phoenix-api/policy/hello-acl", - "phoenix-api/policy/nofile-logging", - "phoenix-api/policy/fault", - "phoenix-api/policy/fault2", - "phoenix-api/policy/delay", - # the pheonix plugins - "plugin/mrpc", - "plugin/mrpclb", - "plugin/rpc_adapter", - "plugin/tcp_rpc_adapter", - "plugin/load_balancer", - # TODO(cjr): Add them back - "plugin/policy/null", - "plugin/policy/ratelimit", - "plugin/policy/qos", - "plugin/policy/logging", - "plugin/policy/hotel-acl", - "plugin/policy/hello-acl-receiver", - "plugin/policy/hello-acl-sender", - "plugin/policy/hello-acl", - "plugin/policy/nofile-logging", - "plugin/policy/fault", - "plugin/policy/fault2", - "plugin/policy/delay", - # examples - "examples/rpc_echo", - "examples/rpc_bench", - "examples/rpc_bench_plus", - "examples/masstree_analytics", - "examples/hotel_reservation", - "examples/load_balancer", - # "examples/hotel_microservices", + "mrpc-build", + "mrpc-derive", + "mrpc-marshal", + "phoenix-api/mrpc", + "phoenix-api/mrpclb", + "phoenix-api/rpc_adapter", + "phoenix-api/tcp_rpc_adapter", + "phoenix-api/load_balancer", + "phoenix-api/policy/null", + "phoenix-api/policy/ratelimit", + "phoenix-api/policy/ratelimit-drop", + "phoenix-api/policy/qos", + "phoenix-api/policy/hotel-acl", + "phoenix-api/policy/logging", + "phoenix-api/policy/hello-acl-receiver", + "phoenix-api/policy/hello-acl-sender", + "phoenix-api/policy/hello-acl", + "phoenix-api/policy/nofile-logging", + "phoenix-api/policy/fault", + "phoenix-api/policy/fault2", + "phoenix-api/policy/delay", + "phoenix-api/policy/admission-control", + "phoenix-api/policy/metrics", + "phoenix-api/policy/mutation", + "phoenix-api/policy/fault-server", + "phoenix-api/policy/logging-server", + "phoenix-api/policy/metrics-server", + "phoenix-api/policy/mutation-server", + "phoenix-api/policy/ratelimit-drop-server", + "plugin/mrpc", + "plugin/mrpclb", + "plugin/rpc_adapter", + "plugin/tcp_rpc_adapter", + "plugin/load_balancer", + "plugin/policy/null", + "plugin/policy/ratelimit", + "plugin/policy/ratelimit-drop", + "plugin/policy/qos", + "plugin/policy/logging", + "plugin/policy/hotel-acl", + "plugin/policy/hello-acl-receiver", + "plugin/policy/hello-acl-sender", + "plugin/policy/hello-acl", + "plugin/policy/nofile-logging", + "plugin/policy/fault", + "plugin/policy/fault2", + "plugin/policy/delay", + "plugin/policy/admission-control", + "plugin/policy/metrics", + "plugin/policy/mutation", + "plugin/policy/fault-server", + "plugin/policy/logging-server", + "plugin/policy/metrics-server", + "plugin/policy/mutation-server", + "plugin/policy/ratelimit-drop-server", + "examples/rpc_echo", + "examples/rpc_bench", + "examples/rpc_bench_plus", + "examples/masstree_analytics", + "examples/hotel_reservation", + "examples/load_balancer", ] exclude = ["3rdparty/prost"] - [workspace.dependencies] -mrpc = { path = "." } -phoenix-api-mrpc = { path = "phoenix-api/mrpc" } -phoenix-api-mrpclb = { path = "phoenix-api/mrpclb" } -phoenix-api-tcp-rpc-adapter = { path = "phoenix-api/tcp_rpc_adapter" } -phoenix-api-rpc-adapter = { path = "phoenix-api/rpc_adapter" } -phoenix-api-load-balancer = { path = "phoenix-api/load_balancer" } -phoenix-api-policy-null = { path = "phoenix-api/policy/null" } -phoenix-api-policy-ratelimit = { path = "phoenix-api/policy/ratelimit" } -phoenix-api-policy-qos = { path = "phoenix-api/policy/qos" } -phoenix-api-policy-hotel-acl = { path = "phoenix-api/policy/hotel-acl" } -phoenix-api-policy-logging = { path = "phoenix-api/policy/logging" } -phoenix-api-policy-hello-acl-receiver = { path = "phoenix-api/policy/hello-acl-receiver" } -phoenix-api-policy-hello-acl-sender = { path = "phoenix-api/policy/hello-acl-sender" } -phoenix-api-policy-hello-acl = { path = "phoenix-api/policy/hello-acl" } -phoenix-api-policy-nofile-logging = { path = "phoenix-api/policy/nofile-logging" } -phoenix-api-policy-fault = { path = "phoenix-api/policy/fault" } -phoenix-api-policy-fault2 = { path = "phoenix-api/policy/fault2" } -phoenix-api-policy-delay = { path = "phoenix-api/policy/delay" } - -mrpc-build = { path = "mrpc-build" } -mrpc-derive = { path = "mrpc-derive" } -mrpc-marshal = { path = "mrpc-marshal" } -prost = { path = "3rdparty/prost" } -prost-build = { path = "3rdparty/prost/prost-build" } -phoenix-mrpc = { path = "plugin/mrpc" } -phoenix-mrpclb = { path = "plugin/mrpclb" } - -phoenix-syscalls = { path = "../../src/phoenix-syscalls" } -phoenix-api = { path = "../../src/phoenix-api" } -ipc = { path = "../../src/ipc" } -mmap = { path = "../../src/mmap" } -rdma = { path = "../../src/rdma" } -shm = { path = "../../src/shm" } -shmalloc = { path = "../../src/shm/shmalloc" } -phoenix_common = { path = "../../src/phoenix_common" } -transport-rdma = { path = "../../src/plugin/transport-rdma", package = "phoenix-transport-rdma" } -transport-tcp = { path = "../../src/plugin/transport-tcp", package = "phoenix-transport-tcp" } -phoenix-salloc = { path = "../../src/plugin/salloc", package = "phoenix-salloc" } -utils = { path = "../../src/utils" } - thiserror = "1.0.31" uuid = "0.8.2" libc = "0.2.103" @@ -153,7 +172,7 @@ futures = "0.3.23" libnuma = "0.0.4" spin = "0.9.3" static_assertions = "1.1.0" -tokio = "1.18.2" # only sync is enabled +tokio = "1.18.2" anyhow = "1.0.58" itertools = "0.10.3" crc32fast = "1.3.2" @@ -164,13 +183,11 @@ proc-macro2 = "1.0.40" md5 = "0.7.0" prettyplease = "0.1.15" toml = "0.5.8" - libloading = "0.7.3" bitvec = "1.0.1" bincode = "1.3.3" socket2 = "0.4.7" slab = "0.4.7" - smol = "1.2.5" structopt = "0.3.23" tracing-subscriber = "0.3" @@ -186,5 +203,146 @@ arc-swap = "1.5.0" crossbeam-utils = "0.8.12" rand = "0.8" +[workspace.dependencies.mrpc] +path = "." + +[workspace.dependencies.phoenix-api-mrpc] +path = "phoenix-api/mrpc" + +[workspace.dependencies.phoenix-api-mrpclb] +path = "phoenix-api/mrpclb" + +[workspace.dependencies.phoenix-api-tcp-rpc-adapter] +path = "phoenix-api/tcp_rpc_adapter" + +[workspace.dependencies.phoenix-api-rpc-adapter] +path = "phoenix-api/rpc_adapter" + +[workspace.dependencies.phoenix-api-load-balancer] +path = "phoenix-api/load_balancer" + +[workspace.dependencies.phoenix-api-policy-null] +path = "phoenix-api/policy/null" + +[workspace.dependencies.phoenix-api-policy-ratelimit] +path = "phoenix-api/policy/ratelimit" + +[workspace.dependencies.phoenix-api-policy-ratelimit-drop] +path = "phoenix-api/policy/ratelimit-drop" + +[workspace.dependencies.phoenix-api-policy-qos] +path = "phoenix-api/policy/qos" + +[workspace.dependencies.phoenix-api-policy-hotel-acl] +path = "phoenix-api/policy/hotel-acl" + +[workspace.dependencies.phoenix-api-policy-logging] +path = "phoenix-api/policy/logging" + +[workspace.dependencies.phoenix-api-policy-hello-acl-receiver] +path = "phoenix-api/policy/hello-acl-receiver" + +[workspace.dependencies.phoenix-api-policy-hello-acl-sender] +path = "phoenix-api/policy/hello-acl-sender" + +[workspace.dependencies.phoenix-api-policy-hello-acl] +path = "phoenix-api/policy/hello-acl" + +[workspace.dependencies.phoenix-api-policy-nofile-logging] +path = "phoenix-api/policy/nofile-logging" + +[workspace.dependencies.phoenix-api-policy-fault] +path = "phoenix-api/policy/fault" + +[workspace.dependencies.phoenix-api-policy-fault2] +path = "phoenix-api/policy/fault2" + +[workspace.dependencies.phoenix-api-policy-delay] +path = "phoenix-api/policy/delay" + +[workspace.dependencies.phoenix-api-policy-admission-control] +path = "phoenix-api/policy/admission-control" + +[workspace.dependencies.phoenix-api-policy-metrics] +path = "phoenix-api/policy/metrics" + +[workspace.dependencies.phoenix-api-policy-mutation] +path = "phoenix-api/policy/mutation" + +[workspace.dependencies.phoenix-api-policy-fault-server] +path = "phoenix-api/policy/fault-server" + +[workspace.dependencies.phoenix-api-policy-logging-server] +path = "phoenix-api/policy/logging-server" + +[workspace.dependencies.phoenix-api-policy-metrics-server] +path = "phoenix-api/policy/metrics-server" + +[workspace.dependencies.phoenix-api-policy-mutation-server] +path = "phoenix-api/policy/mutation-server" + +[workspace.dependencies.phoenix-api-policy-ratelimit-drop-server] +path = "phoenix-api/policy/ratelimit-drop-server" + +[workspace.dependencies.mrpc-build] +path = "mrpc-build" + +[workspace.dependencies.mrpc-derive] +path = "mrpc-derive" + +[workspace.dependencies.mrpc-marshal] +path = "mrpc-marshal" + +[workspace.dependencies.prost] +path = "3rdparty/prost" + +[workspace.dependencies.prost-build] +path = "3rdparty/prost/prost-build" + +[workspace.dependencies.phoenix-mrpc] +path = "plugin/mrpc" + +[workspace.dependencies.phoenix-mrpclb] +path = "plugin/mrpclb" + +[workspace.dependencies.phoenix-syscalls] +path = "../../src/phoenix-syscalls" + +[workspace.dependencies.phoenix-api] +path = "../../src/phoenix-api" + +[workspace.dependencies.ipc] +path = "../../src/ipc" + +[workspace.dependencies.mmap] +path = "../../src/mmap" + +[workspace.dependencies.rdma] +path = "../../src/rdma" + +[workspace.dependencies.shm] +path = "../../src/shm" + +[workspace.dependencies.shmalloc] +path = "../../src/shm/shmalloc" + +[workspace.dependencies.phoenix_common] +path = "../../src/phoenix_common" + +[workspace.dependencies.transport-rdma] +path = "../../src/plugin/transport-rdma" +package = "phoenix-transport-rdma" + +[workspace.dependencies.transport-tcp] +path = "../../src/plugin/transport-tcp" +package = "phoenix-transport-tcp" + +[workspace.dependencies.phoenix-salloc] +path = "../../src/plugin/salloc" +package = "phoenix-salloc" + +[workspace.dependencies.utils] +path = "../../src/utils" + [profile.release] debug = true diff --git a/experimental/mrpc/Makefile.toml b/experimental/mrpc/Makefile.toml index 1be09122..e72a9e13 100644 --- a/experimental/mrpc/Makefile.toml +++ b/experimental/mrpc/Makefile.toml @@ -11,13 +11,35 @@ You can customize your own developmenet testing flow. Build and deploy mRPC plugins. Build and deploy phoenix plugins. Run phoenix daemon. ''' dependencies = [ - "build-phoenixos", + # "build-phoenixos", "build-mrpc-plugins", "build-plugins", "deploy-mrpc-plugins", "build-mrpc-examples", "run-phoenixos", ] +[tasks.build-mrpc-plugin-single] +description = ''' +Build specified plugins by package name. +If no argument is provided, build all plugins in this workspace. +''' +# cargo make build-plugins [package-name]* +command = "${TARGET_DIR}/release/phoenix_cargo" +args = [ + "--compile-log", + "${PHOENIX_COMPILE_LOG}", + "--host-dep", + "${HOST_DEP}", + "--", + "build", + "--release", + "--target-dir", + "${PHOENIX_TARGET_DIR}", + "--manifest-path", + "${PROJECT_ROOT}/experimental/mrpc/Cargo.toml", + "-p", + "phoenix-${@}", +] [tasks.build-mrpc-plugins] description = ''' diff --git a/experimental/mrpc/load-mrpc-plugins.toml b/experimental/mrpc/load-mrpc-plugins.toml index 6aeafef5..8b9cea2c 100644 --- a/experimental/mrpc/load-mrpc-plugins.toml +++ b/experimental/mrpc/load-mrpc-plugins.toml @@ -81,6 +81,24 @@ lib_path = "plugins/libphoenix_hello_acl_sender.rlib" config_string = ''' ''' +[[addons]] +name = "AdmissionControl" +lib_path = "plugins/libphoenix_admission_control.rlib" +config_string = ''' +''' + +[[addons]] +name = "Metrics" +lib_path = "plugins/libphoenix_metrics.rlib" +config_string = ''' +''' + +[[addons]] +name = "Mutation" +lib_path = "plugins/libphoenix_mutation.rlib" +config_string = ''' +''' + [[addons]] name = "HelloAcl" lib_path = "plugins/libphoenix_hello_acl.rlib" @@ -112,3 +130,43 @@ config_string = ''' delay_probability = 0.2 delay_ms = 100 ''' + +[[addons]] +name = "RateLimitDrop" +lib_path = "plugins/libphoenix_ratelimit_drop.rlib" +config_string = ''' +requests_per_sec = 4 +bucket_size = 1000 +''' + +[[addons]] +name = "FaultServer" +lib_path = "plugins/libphoenix_fault_server.rlib" +config_string = ''' +''' + +[[addons]] +name = "LoggingServer" +lib_path = "plugins/libphoenix_logging_server.rlib" +config_string = ''' +''' + +[[addons]] +name = "MutationServer" +lib_path = "plugins/libphoenix_mutation_server.rlib" +config_string = ''' +''' + +[[addons]] +name = "MetricsServer" +lib_path = "plugins/libphoenix_metrics_server.rlib" +config_string = ''' +''' + +[[addons]] +name = "RateLimitDropServer" +lib_path = "plugins/libphoenix_ratelimit_drop_server.rlib" +config_string = ''' +requests_per_sec = 4 +bucket_size = 1000 +''' diff --git a/experimental/mrpc/phoenix-api/policy/admission-control/Cargo.toml b/experimental/mrpc/phoenix-api/policy/admission-control/Cargo.toml new file mode 100644 index 00000000..c94e4fc1 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/admission-control/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-admission-control" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/admission-control/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/admission-control/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/admission-control/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/admission-control/src/lib.rs b/experimental/mrpc/phoenix-api/policy/admission-control/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/admission-control/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/fault-server/Cargo.toml b/experimental/mrpc/phoenix-api/policy/fault-server/Cargo.toml new file mode 100644 index 00000000..b5b389cb --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault-server/Cargo.toml @@ -0,0 +1,14 @@ + +[package] +name = "phoenix-api-policy-fault-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/fault-server/logging/Cargo.toml b/experimental/mrpc/phoenix-api/policy/fault-server/logging/Cargo.toml new file mode 100644 index 00000000..60d4a659 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault-server/logging/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-logging" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/fault-server/logging/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/fault-server/logging/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault-server/logging/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/fault-server/logging/src/lib.rs b/experimental/mrpc/phoenix-api/policy/fault-server/logging/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault-server/logging/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/fault-server/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/fault-server/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault-server/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/fault-server/src/lib.rs b/experimental/mrpc/phoenix-api/policy/fault-server/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/fault-server/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/file-logging/Cargo.toml b/experimental/mrpc/phoenix-api/policy/file-logging/Cargo.toml new file mode 100644 index 00000000..9240b465 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/file-logging/Cargo.toml @@ -0,0 +1,14 @@ + +[package] +name = "phoenix-api-policy-file-logging" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/file-logging/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/file-logging/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/file-logging/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/file-logging/src/lib.rs b/experimental/mrpc/phoenix-api/policy/file-logging/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/file-logging/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/logging-server/Cargo.toml b/experimental/mrpc/phoenix-api/policy/logging-server/Cargo.toml new file mode 100644 index 00000000..fcfed524 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/logging-server/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-logging-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/logging-server/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/logging-server/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/logging-server/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/logging-server/src/lib.rs b/experimental/mrpc/phoenix-api/policy/logging-server/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/logging-server/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/metrics-server/Cargo.toml b/experimental/mrpc/phoenix-api/policy/metrics-server/Cargo.toml new file mode 100644 index 00000000..8a0d5c63 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/metrics-server/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-metrics-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/metrics-server/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/metrics-server/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/metrics-server/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/metrics-server/src/lib.rs b/experimental/mrpc/phoenix-api/policy/metrics-server/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/metrics-server/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/metrics/Cargo.toml b/experimental/mrpc/phoenix-api/policy/metrics/Cargo.toml new file mode 100644 index 00000000..b7d3bbbb --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/metrics/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-metrics" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/metrics/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/metrics/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/metrics/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/metrics/src/lib.rs b/experimental/mrpc/phoenix-api/policy/metrics/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/metrics/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/mutation-server/Cargo.toml b/experimental/mrpc/phoenix-api/policy/mutation-server/Cargo.toml new file mode 100644 index 00000000..8baea7c9 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/mutation-server/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-mutation-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/mutation-server/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/mutation-server/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/mutation-server/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/mutation-server/src/lib.rs b/experimental/mrpc/phoenix-api/policy/mutation-server/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/mutation-server/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/mutation/Cargo.toml b/experimental/mrpc/phoenix-api/policy/mutation/Cargo.toml new file mode 100644 index 00000000..b17eaabd --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/mutation/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-mutation" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/mutation/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/mutation/src/control_plane.rs new file mode 100644 index 00000000..4778cd75 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/mutation/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/mutation/src/lib.rs b/experimental/mrpc/phoenix-api/policy/mutation/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/mutation/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/Cargo.toml b/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/Cargo.toml new file mode 100644 index 00000000..211984a9 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-ratelimit-drop-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/src/control_plane.rs new file mode 100644 index 00000000..dc317e5f --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(u64, u64), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/src/lib.rs b/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/ratelimit-drop-server/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/phoenix-api/policy/ratelimit-drop/Cargo.toml b/experimental/mrpc/phoenix-api/policy/ratelimit-drop/Cargo.toml new file mode 100644 index 00000000..743b0d4b --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/ratelimit-drop/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "phoenix-api-policy-ratelimit-drop" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix-api.workspace = true + +serde.workspace = true diff --git a/experimental/mrpc/phoenix-api/policy/ratelimit-drop/src/control_plane.rs b/experimental/mrpc/phoenix-api/policy/ratelimit-drop/src/control_plane.rs new file mode 100644 index 00000000..dc317e5f --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/ratelimit-drop/src/control_plane.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +type IResult = Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Request { + NewConfig(u64, u64), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ResponseKind {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Response(pub IResult); diff --git a/experimental/mrpc/phoenix-api/policy/ratelimit-drop/src/lib.rs b/experimental/mrpc/phoenix-api/policy/ratelimit-drop/src/lib.rs new file mode 100644 index 00000000..412c5a55 --- /dev/null +++ b/experimental/mrpc/phoenix-api/policy/ratelimit-drop/src/lib.rs @@ -0,0 +1 @@ +pub mod control_plane; diff --git a/experimental/mrpc/plugin/policy/admission-control/Cargo.toml b/experimental/mrpc/plugin/policy/admission-control/Cargo.toml new file mode 100644 index 00000000..3e776fc8 --- /dev/null +++ b/experimental/mrpc/plugin/policy/admission-control/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "phoenix-admission-control" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + + +[dependencies] +phoenix-api-policy-admission-control.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true + +phoenix_common.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +fnv.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/admission-control/src/config.rs b/experimental/mrpc/plugin/policy/admission-control/src/config.rs new file mode 100644 index 00000000..11fe557c --- /dev/null +++ b/experimental/mrpc/plugin/policy/admission-control/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AdmissionControlConfig {} + +impl AdmissionControlConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/admission-control/src/engine.rs b/experimental/mrpc/plugin/policy/admission-control/src/engine.rs new file mode 100644 index 00000000..4e81ed6e --- /dev/null +++ b/experimental/mrpc/plugin/policy/admission-control/src/engine.rs @@ -0,0 +1,245 @@ +//! This engine can only be placed at the sender side for now. +use anyhow::{anyhow, Result}; +use fnv::FnvHashMap as HashMap; +use futures::future::BoxFuture; +use rand::Rng; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; +use std::time::Instant; + +use phoenix_api::rpc::{RpcId, StatusCode, TransportStatus}; +use phoenix_api_policy_admission_control::control_plane; + +use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::AdmissionControlConfig; + +pub mod hello { + // The string specified here must match the proto package name + include!("rpc_hello.rs"); +} + +pub(crate) struct AdmissionControlEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + pub(crate) total: u32, + pub(crate) success: u32, + pub(crate) last_ts: Instant, + pub(crate) config: AdmissionControlConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for AdmissionControlEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "AdmissionControlEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + // Update config + self.config = AdmissionControlConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(AdmissionControlEngine, node); + +impl Decompose for AdmissionControlEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + + let mut collections = ResourceCollection::with_capacity(2); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert("success".to_string(), Box::new(engine.success as u32)); + collections.insert("total".to_string(), Box::new(engine.total as u32)); + collections.insert("last_ts".to_string(), Box::new(engine.last_ts)); + (collections, engine.node) + } +} + +impl AdmissionControlEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let total = *local + .remove("total") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let success = *local + .remove("success") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let last_ts = *local + .remove("last_ts") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + + let engine = AdmissionControlEngine { + node, + indicator: Default::default(), + total, + success, + last_ts, + config, + }; + Ok(engine) + } +} + +impl AdmissionControlEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +#[inline] +fn calculate_reject_probability(total: u32, succ: u32, threhold: u32, agg: f32) -> f32 { + let s: f32 = succ as f32 / threhold as f32; + let u = (total as f32 - s) / (total as f32 + 1.0); + u.powf(1.0 / agg) +} + +impl AdmissionControlEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let conn_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.conn_id; + let call_id = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }.call_id; + let rpc_id = RpcId { + 0: conn_id, + 1: call_id, + }; + if rand::random::() + < calculate_reject_probability( + self.total as u32, + self.success as u32, + 10, + 0.5, + ) + { + let error = EngineRxMessage::Ack( + rpc_id, + TransportStatus::Error(unsafe { NonZeroU32::new_unchecked(403) }), + ); + self.rx_outputs()[0].send(error).unwrap_or_else(|e| { + log::warn!("error when bubbling up the error, send failed e: {}", e) + }); + } else { + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; + } + } + // XXX TODO(cjr): it is best not to reorder the message + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + // forward all rx msgs + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + match m { + EngineRxMessage::RpcMessage(msg) => { + let meta = unsafe { &*msg.meta.as_ptr() }; + if meta.status_code == StatusCode::Success { + self.success += 1; + } + self.total += 1; + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => { + self.rx_outputs()[0].send(m)?; + } + }; + if std::time::Instant::now() - self.last_ts > std::time::Duration::from_secs(5) { + self.last_ts = std::time::Instant::now(); + self.total = 0; + self.success = 0; + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/admission-control/src/lib.rs b/experimental/mrpc/plugin/policy/admission-control/src/lib.rs new file mode 100644 index 00000000..dc7af5a9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/admission-control/src/lib.rs @@ -0,0 +1,34 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] + +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::AdmissionControlConfig; +use crate::module::AdmissionControlAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = AdmissionControlConfig::new(config_string)?; + let addon = AdmissionControlAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/admission-control/src/module.rs b/experimental/mrpc/plugin/policy/admission-control/src/module.rs new file mode 100644 index 00000000..1afc5ce7 --- /dev/null +++ b/experimental/mrpc/plugin/policy/admission-control/src/module.rs @@ -0,0 +1,103 @@ +use anyhow::{bail, Result}; +use fnv::FnvHashMap as HashMap; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::AdmissionControlEngine; +use crate::config::AdmissionControlConfig; + +pub(crate) struct AdmissionControlEngineBuilder { + node: DataPathNode, + config: AdmissionControlConfig, +} + +impl AdmissionControlEngineBuilder { + fn new(node: DataPathNode, config: AdmissionControlConfig) -> Self { + AdmissionControlEngineBuilder { node, config } + } + + fn build(self) -> Result { + Ok(AdmissionControlEngine { + node: self.node, + indicator: Default::default(), + total: 0, + success: 0, + last_ts: std::time::Instant::now(), + config: self.config, + }) + } +} + +pub struct AdmissionControlAddon { + config: AdmissionControlConfig, +} + +impl AdmissionControlAddon { + pub const ADMISSION_CONTROL_ENGINE: EngineType = EngineType("AdmissionControlEngine"); + pub const ENGINES: &'static [EngineType] = &[AdmissionControlAddon::ADMISSION_CONTROL_ENGINE]; +} + +impl AdmissionControlAddon { + pub fn new(config: AdmissionControlConfig) -> Self { + AdmissionControlAddon { config } + } +} + +impl PhoenixAddon for AdmissionControlAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + AdmissionControlAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != AdmissionControlAddon::ADMISSION_CONTROL_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = AdmissionControlEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != AdmissionControlAddon::ADMISSION_CONTROL_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = AdmissionControlEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/admission-control/src/rpc_hello.rs b/experimental/mrpc/plugin/policy/admission-control/src/rpc_hello.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/admission-control/src/rpc_hello.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/fault-server/Cargo.toml b/experimental/mrpc/plugin/policy/fault-server/Cargo.toml new file mode 100644 index 00000000..ad5454f2 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault-server/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-fault-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-fault-server.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/fault-server/src/config.rs b/experimental/mrpc/plugin/policy/fault-server/src/config.rs new file mode 100644 index 00000000..fa8605e5 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault-server/src/config.rs @@ -0,0 +1,37 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FaultServerConfig {} + +impl FaultServerConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/fault-server/src/engine.rs b/experimental/mrpc/plugin/policy/fault-server/src/engine.rs new file mode 100644 index 00000000..b55b016d --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault-server/src/engine.rs @@ -0,0 +1,248 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, StatusCode, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; + +use phoenix_api_policy_fault_server::control_plane; +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; +use phoenix_common::engine::datapath::meta_pool::MetaBufferPool; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::{RpcMessageRx, RpcMessageTx}; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, FaultServerConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub(crate) struct FaultServerEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: FaultServerConfig, + pub(crate) meta_buf_pool: MetaBufferPool, + pub(crate) var_probability: f32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for FaultServerEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "FaultServerEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = FaultServerConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(FaultServerEngine, node); + +impl Decompose for FaultServerEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert("meta_buf_pool".to_string(), Box::new(engine.meta_buf_pool)); + (collections, engine.node) + } +} + +impl FaultServerEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let meta_buf_pool = *local + .remove("meta_buf_pool") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let var_probability = 0.01; + + let engine = FaultServerEngine { + node, + indicator: Default::default(), + config, + meta_buf_pool, + var_probability, + }; + Ok(engine) + } +} + +impl FaultServerEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +/// Copy the RPC request to a private heap and returns the request. +#[inline] +fn materialize_rx(msg: &RpcMessageRx) -> Box { + let req_ptr = Unique::new(msg.addr_backend as *mut hello::HelloRequest).unwrap(); + let req = unsafe { req_ptr.as_ref() }; + // returns a private_req + Box::new(req.clone()) +} + +impl FaultServerEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + self.tx_outputs()[0].send(msg)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + match m { + EngineRxMessage::Ack(rpc_id, status) => { + if let Ok(()) = self.meta_buf_pool.release(rpc_id) { + // log::info!( + // "Access denied ack received, rpc_id: {:?} metabuf released", + // rpc_id + // ); + } else { + // log::info!("release failed!: {:?}", rpc_id); + self.rx_outputs()[0].send(m)?; + } + } + EngineRxMessage::RpcMessage(msg) => { + let private_req = materialize_rx(&msg); + if rand::random::() < self.var_probability { + // We need to copy meta, add it to meta_buf_pool, and send it as the tx msg + // Is there better way to do this and avoid unsafe? + let mut meta = unsafe { msg.meta.as_ref().clone() }; + meta.status_code = StatusCode::AccessDenied; + let mut meta_ptr = self + .meta_buf_pool + .obtain(RpcId(meta.conn_id, meta.call_id)) + .expect("meta_buf_pool is full"); + unsafe { + meta_ptr.as_meta_ptr().write(meta); + meta_ptr.0.as_mut().num_sge = 0; + meta_ptr.0.as_mut().value_len = 0; + } + let rpc_msg = RpcMessageTx { + meta_buf_ptr: meta_ptr, + addr_backend: 0, + }; + let new_msg = EngineTxMessage::RpcMessage(rpc_msg); + self.tx_outputs()[0] + .send(new_msg) + .expect("send new message error"); + let msg_call_ids = + [meta.call_id, meta.call_id, meta.call_id, meta.call_id]; + self.tx_outputs()[0].send(EngineTxMessage::ReclaimRecvBuf( + meta.conn_id, + msg_call_ids, + ))?; + } else { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + } + EngineRxMessage::RecvError(_, _) => { + self.rx_outputs()[0].send(m)?; + } + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/fault-server/src/lib.rs b/experimental/mrpc/plugin/policy/fault-server/src/lib.rs new file mode 100644 index 00000000..b833dc14 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault-server/src/lib.rs @@ -0,0 +1,37 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::FaultServerConfig; +use crate::module::FaultServerAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = FaultServerConfig::new(config_string)?; + let addon = FaultServerAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/fault-server/src/module.rs b/experimental/mrpc/plugin/policy/fault-server/src/module.rs new file mode 100644 index 00000000..a950423e --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault-server/src/module.rs @@ -0,0 +1,107 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use super::engine::FaultServerEngine; +use crate::config::{create_log_file, FaultServerConfig}; +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::meta_pool::MetaBufferPool; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub(crate) struct FaultServerEngineBuilder { + node: DataPathNode, + config: FaultServerConfig, +} + +impl FaultServerEngineBuilder { + fn new(node: DataPathNode, config: FaultServerConfig) -> Self { + FaultServerEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let var_probability = 0.01; + const META_BUFFER_POOL_CAP: usize = 128; + Ok(FaultServerEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + meta_buf_pool: MetaBufferPool::new(META_BUFFER_POOL_CAP), + var_probability, + }) + } +} + +pub struct FaultServerAddon { + config: FaultServerConfig, +} + +impl FaultServerAddon { + pub const FAULT_SERVER_ENGINE: EngineType = EngineType("FaultServerEngine"); + pub const ENGINES: &'static [EngineType] = &[FaultServerAddon::FAULT_SERVER_ENGINE]; +} + +impl FaultServerAddon { + pub fn new(config: FaultServerConfig) -> Self { + FaultServerAddon { config } + } +} + +impl PhoenixAddon for FaultServerAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + FaultServerAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != FaultServerAddon::FAULT_SERVER_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = FaultServerEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != FaultServerAddon::FAULT_SERVER_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = FaultServerEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/fault-server/src/proto.rs b/experimental/mrpc/plugin/policy/fault-server/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/fault-server/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/file-logging/Cargo.toml b/experimental/mrpc/plugin/policy/file-logging/Cargo.toml new file mode 100644 index 00000000..ab006d87 --- /dev/null +++ b/experimental/mrpc/plugin/policy/file-logging/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-file-logging" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-file-logging.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/file-logging/src/config.rs b/experimental/mrpc/plugin/policy/file-logging/src/config.rs new file mode 100644 index 00000000..d4094d82 --- /dev/null +++ b/experimental/mrpc/plugin/policy/file-logging/src/config.rs @@ -0,0 +1,39 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +use crate::engine::struct_rpc_events_file; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FileLoggingConfig {} + +impl FileLoggingConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/file-logging/src/engine.rs b/experimental/mrpc/plugin/policy/file-logging/src/engine.rs new file mode 100644 index 00000000..82168599 --- /dev/null +++ b/experimental/mrpc/plugin/policy/file-logging/src/engine.rs @@ -0,0 +1,267 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_api_policy_file_logging::control_plane; + +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::RpcMessageTx; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, FileLoggingConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub mod hello { + include!("proto.rs"); +} + +pub struct struct_rpc_events_file { + pub timestamp: DateTime, + pub event_type: String, + pub source: String, + pub destination: String, + pub rpc: String, +} +impl struct_rpc_events_file { + pub fn new( + timestamp: DateTime, + event_type: String, + source: String, + destination: String, + rpc: String, + ) -> struct_rpc_events_file { + struct_rpc_events_file { + timestamp: timestamp, + event_type: event_type, + source: source, + destination: destination, + rpc: rpc, + } + } +} + +impl fmt::Display for struct_rpc_events_file { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.timestamp); + write!(f, "{}", self.event_type); + write!(f, "{}", self.source); + write!(f, "{}", self.destination); + write!(f, "{}", self.rpc); + write!(f, "\n") + } +} + +pub(crate) struct FileLoggingEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: FileLoggingConfig, + pub(crate) file_rpc_events_file: File, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for FileLoggingEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "FileLoggingEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = FileLoggingConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(FileLoggingEngine, node); + +impl Decompose for FileLoggingEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl FileLoggingEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let mut file_rpc_events_file = create_log_file(); + let engine = FileLoggingEngine { + node, + indicator: Default::default(), + config, + file_rpc_events_file, + }; + Ok(engine) + } +} + +impl FileLoggingEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl FileLoggingEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let meta_ref = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }; + let mut input = Vec::new(); + input.push(msg); + for event in input + .iter() + .map(|req| { + struct_rpc_events_file::new( + Utc::now(), + format!("{:?}", meta_ref.msg_type), + format!("{:?}", meta_ref.conn_id), + format!("{:?}", meta_ref.conn_id), + format!("{}", req.addr_backend.clone()), + ) + }) + .collect::>() + { + write!(self.file_rpc_events_file, "{}", event); + } + let output: Vec<_> = input + .iter() + .map(|req| { + RpcMessageGeneral::TxMessage(EngineTxMessage::RpcMessage( + RpcMessageTx::new( + req.meta_buf_ptr.clone(), + req.addr_backend.clone(), + ), + )) + }) + .collect::>(); + + for msg in output { + match msg { + RpcMessageGeneral::TxMessage(msg) => { + self.tx_outputs()[0].send(msg)?; + } + RpcMessageGeneral::RxMessage(msg) => { + self.rx_outputs()[0].send(msg)?; + } + _ => {} + } + } + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + // todo + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/file-logging/src/lib.rs b/experimental/mrpc/plugin/policy/file-logging/src/lib.rs new file mode 100644 index 00000000..64ba3b48 --- /dev/null +++ b/experimental/mrpc/plugin/policy/file-logging/src/lib.rs @@ -0,0 +1,39 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use crate::engine::struct_rpc_events_file; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::FileLoggingConfig; +use crate::module::FileLoggingAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = FileLoggingConfig::new(config_string)?; + let addon = FileLoggingAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/file-logging/src/module.rs b/experimental/mrpc/plugin/policy/file-logging/src/module.rs new file mode 100644 index 00000000..ab5ce3bd --- /dev/null +++ b/experimental/mrpc/plugin/policy/file-logging/src/module.rs @@ -0,0 +1,106 @@ +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::FileLoggingEngine; +use crate::config::{create_log_file, FileLoggingConfig}; +use crate::engine::struct_rpc_events_file; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub(crate) struct FileLoggingEngineBuilder { + node: DataPathNode, + config: FileLoggingConfig, +} + +impl FileLoggingEngineBuilder { + fn new(node: DataPathNode, config: FileLoggingConfig) -> Self { + FileLoggingEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let mut file_rpc_events_file = create_log_file(); + Ok(FileLoggingEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + file_rpc_events_file, + }) + } +} + +pub struct FileLoggingAddon { + config: FileLoggingConfig, +} + +impl FileLoggingAddon { + pub const FILE_LOGGING_ENGINE: EngineType = EngineType("FileLoggingEngine"); + pub const ENGINES: &'static [EngineType] = &[FileLoggingAddon::FILE_LOGGING_ENGINE]; +} + +impl FileLoggingAddon { + pub fn new(config: FileLoggingConfig) -> Self { + FileLoggingAddon { config } + } +} + +impl PhoenixAddon for FileLoggingAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + FileLoggingAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != FileLoggingAddon::FILE_LOGGING_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = FileLoggingEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != FileLoggingAddon::FILE_LOGGING_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = FileLoggingEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/file-logging/src/proto.rs b/experimental/mrpc/plugin/policy/file-logging/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/file-logging/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/logging-server/Cargo.toml b/experimental/mrpc/plugin/policy/logging-server/Cargo.toml new file mode 100644 index 00000000..d510ca4d --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging-server/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "phoenix-logging-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-logging-server.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/logging-server/src/config.rs b/experimental/mrpc/plugin/policy/logging-server/src/config.rs new file mode 100644 index 00000000..f6b8c2f9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging-server/src/config.rs @@ -0,0 +1,41 @@ +//! template file for engine config +//! we can add custome functions here +//! so that the whole crate can import the function + +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +/// currently, logging engine does not need a config +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LoggingServerConfig {} + +impl LoggingServerConfig { + /// Get config from toml file + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} + +/// Create a log file in `/tmp/phoenix/log` +/// This function will be called every time +/// a logging engine is started or restored +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/logging-server/src/engine.rs b/experimental/mrpc/plugin/policy/logging-server/src/engine.rs new file mode 100644 index 00000000..a1f4fbde --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging-server/src/engine.rs @@ -0,0 +1,207 @@ +//! main logic happens here +use anyhow::{anyhow, Result}; +use chrono::Utc; +use futures::future::BoxFuture; +use phoenix_api_policy_logging_server::control_plane; +use phoenix_common::engine::datapath::RpcMessageTx; +use std::io::Write; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage}; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, LoggingServerConfig}; + +pub mod hello { + include!("proto.rs"); +} + +/// The internal state of an logging engine, +/// it contains some template fields like `node`, `indicator`, +/// a config field, in that case `LoggingServerConfig` +/// and other custome fields like `log_file +pub(crate) struct LoggingServerEngine { + pub(crate) node: DataPathNode, + pub(crate) indicator: Indicator, + pub(crate) config: LoggingServerConfig, + /// log_file is where the log will be written into + /// it is temperoray, i.e. we don't store it when restart + pub(crate) log_file: std::fs::File, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +/// template +impl Engine for LoggingServerEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "LoggingServerEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + self.config = LoggingServerConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(LoggingServerEngine, node); + +impl Decompose for LoggingServerEngine { + /// flush will be called when we need to clean the transient state before decompose + /// # return + /// * `Result` - number of work drained from tx & rx queue + fn flush(&mut self) -> Result { + let mut work = 0; + /// drain the rx & tx queue + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + self.log_file.flush()?; + // file will automatically be closed when the engine is dropped + Ok(work) + } + + /// template + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl LoggingServerEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + + let log_file = create_log_file(); + let engine = LoggingServerEngine { + node, + indicator: Default::default(), + config, + log_file, + }; + Ok(engine) + } +} + +impl LoggingServerEngine { + async fn mainloop(&mut self) -> EngineResult { + // open a write buffer to a file + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl LoggingServerEngine { + /// main logic about handling rx & tx input messages + /// note that a logging engine can be deployed in client-side or server-side + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + self.tx_outputs()[0].send(msg)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + match m { + EngineRxMessage::Ack(rpc_id, _status) => { + self.rx_outputs()[0].send(m)?; + } + EngineRxMessage::RpcMessage(msg) => { + let meta_ref = unsafe { msg.meta.as_ref() }; + // write the metadata into the file + // since meta_ref implements Debug, we can use {:?} + // rather than manully parse the metadata struct + write!( + self.log_file, + "{}{}{}{}\n", + Utc::now(), + format!("{:?}", meta_ref.msg_type), + format!("{:?}", meta_ref.conn_id), + format!("{:?}", meta_ref.conn_id), + ) + .unwrap(); + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + EngineRxMessage::RecvError(_, _) => { + self.rx_outputs()[0].send(m)?; + } + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/logging-server/src/lib.rs b/experimental/mrpc/plugin/policy/logging-server/src/lib.rs new file mode 100644 index 00000000..cfcbc595 --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging-server/src/lib.rs @@ -0,0 +1,33 @@ +//! template file for export +#![feature(ptr_internals)] +#![feature(peer_credentials_unix_socket)] +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::LoggingServerConfig; +use crate::module::LoggingServerAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = LoggingServerConfig::new(config_string)?; + let addon = LoggingServerAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/logging-server/src/module.rs b/experimental/mrpc/plugin/policy/logging-server/src/module.rs new file mode 100644 index 00000000..31fd42fb --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging-server/src/module.rs @@ -0,0 +1,105 @@ +//! template file for an engine +//! we only need to change the args in config and engine constructor + +use anyhow::{bail, Result}; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::LoggingServerEngine; +use crate::config::{create_log_file, LoggingServerConfig}; + +pub(crate) struct LoggingServerEngineBuilder { + node: DataPathNode, + config: LoggingServerConfig, +} + +impl LoggingServerEngineBuilder { + fn new(node: DataPathNode, config: LoggingServerConfig) -> Self { + LoggingServerEngineBuilder { node, config } + } + + fn build(self) -> Result { + let log_file = create_log_file(); + + Ok(LoggingServerEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + log_file, + }) + } +} + +pub struct LoggingServerAddon { + config: LoggingServerConfig, +} + +impl LoggingServerAddon { + pub const LOGGING_ENGINE: EngineType = EngineType("LoggingServerEngine"); + pub const ENGINES: &'static [EngineType] = &[LoggingServerAddon::LOGGING_ENGINE]; +} + +impl LoggingServerAddon { + pub fn new(config: LoggingServerConfig) -> Self { + LoggingServerAddon { config } + } +} + +impl PhoenixAddon for LoggingServerAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + LoggingServerAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != LoggingServerAddon::LOGGING_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = LoggingServerEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != LoggingServerAddon::LOGGING_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = LoggingServerEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/logging-server/src/proto.rs b/experimental/mrpc/plugin/policy/logging-server/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/logging-server/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/metrics-server/Cargo.toml b/experimental/mrpc/plugin/policy/metrics-server/Cargo.toml new file mode 100644 index 00000000..560932f9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics-server/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "phoenix-metrics-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + + +[dependencies] +phoenix-api-policy-metrics-server.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true + +phoenix_common.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +fnv.workspace = true diff --git a/experimental/mrpc/plugin/policy/metrics-server/src/config.rs b/experimental/mrpc/plugin/policy/metrics-server/src/config.rs new file mode 100644 index 00000000..eca26592 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics-server/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricsServerConfig {} + +impl MetricsServerConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/metrics-server/src/engine.rs b/experimental/mrpc/plugin/policy/metrics-server/src/engine.rs new file mode 100644 index 00000000..22f6ea1a --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics-server/src/engine.rs @@ -0,0 +1,191 @@ +//! This engine can only be placed at the sender side for now. +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; + +use anyhow::{anyhow, Result}; +use fnv::FnvHashMap as HashMap; +use futures::future::BoxFuture; + +use phoenix_api::rpc::{RpcId, StatusCode, TransportStatus}; +use phoenix_api_policy_metrics_server::control_plane; + +use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::MetricsServerConfig; + +pub(crate) struct MetricsServerEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + pub(crate) num_succ: u32, + pub(crate) num_rej: u32, + + pub(crate) config: MetricsServerConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for MetricsServerEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "MetricsServerEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + // Update config + self.config = MetricsServerConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(MetricsServerEngine, node); + +impl Decompose for MetricsServerEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + + let mut collections = ResourceCollection::with_capacity(2); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert("num_rej".to_string(), Box::new(engine.num_rej)); + collections.insert("num_succ".to_string(), Box::new(engine.num_succ)); + (collections, engine.node) + } +} + +impl MetricsServerEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let num_succ = *local + .remove("num_succ") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let num_rej = *local + .remove("num_rej") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let engine = MetricsServerEngine { + node, + indicator: Default::default(), + num_succ, + num_rej, + config, + }; + Ok(engine) + } +} + +impl MetricsServerEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +impl MetricsServerEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let meta = unsafe { &*msg.meta_buf_ptr.as_meta_ptr() }; + if meta.status_code == StatusCode::Success { + self.num_succ += 1; + } else { + self.num_rej += 1; + } + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; + } + m => { + self.tx_outputs()[0].send(m)?; + } + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + // forward all rx msgs + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + self.rx_outputs()[0].send(m)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/metrics-server/src/lib.rs b/experimental/mrpc/plugin/policy/metrics-server/src/lib.rs new file mode 100644 index 00000000..c6a5d526 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics-server/src/lib.rs @@ -0,0 +1,34 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] + +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::MetricsServerConfig; +use crate::module::MetricsServerAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = MetricsServerConfig::new(config_string)?; + let addon = MetricsServerAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/metrics-server/src/module.rs b/experimental/mrpc/plugin/policy/metrics-server/src/module.rs new file mode 100644 index 00000000..e20b9246 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics-server/src/module.rs @@ -0,0 +1,102 @@ +use anyhow::{bail, Result}; +use fnv::FnvHashMap as HashMap; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::MetricsServerEngine; +use crate::config::MetricsServerConfig; + +pub(crate) struct MetricsServerEngineBuilder { + node: DataPathNode, + config: MetricsServerConfig, +} + +impl MetricsServerEngineBuilder { + fn new(node: DataPathNode, config: MetricsServerConfig) -> Self { + MetricsServerEngineBuilder { node, config } + } + + fn build(self) -> Result { + Ok(MetricsServerEngine { + node: self.node, + indicator: Default::default(), + num_succ: 0, + num_rej: 0, + config: self.config, + }) + } +} + +pub struct MetricsServerAddon { + config: MetricsServerConfig, +} + +impl MetricsServerAddon { + pub const METRICS_ENGINE: EngineType = EngineType("MetricsServerEngine"); + pub const ENGINES: &'static [EngineType] = &[MetricsServerAddon::METRICS_ENGINE]; +} + +impl MetricsServerAddon { + pub fn new(config: MetricsServerConfig) -> Self { + MetricsServerAddon { config } + } +} + +impl PhoenixAddon for MetricsServerAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + MetricsServerAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != MetricsServerAddon::METRICS_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = MetricsServerEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != MetricsServerAddon::METRICS_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = MetricsServerEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/metrics/Cargo.toml b/experimental/mrpc/plugin/policy/metrics/Cargo.toml new file mode 100644 index 00000000..b5118932 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "phoenix-metrics" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + + +[dependencies] +phoenix-api-policy-metrics.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true + +phoenix_common.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +fnv.workspace = true diff --git a/experimental/mrpc/plugin/policy/metrics/src/config.rs b/experimental/mrpc/plugin/policy/metrics/src/config.rs new file mode 100644 index 00000000..d50cca93 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricsConfig {} + +impl MetricsConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/metrics/src/engine.rs b/experimental/mrpc/plugin/policy/metrics/src/engine.rs new file mode 100644 index 00000000..4cf93363 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics/src/engine.rs @@ -0,0 +1,191 @@ +//! This engine can only be placed at the sender side for now. +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; + +use anyhow::{anyhow, Result}; +use fnv::FnvHashMap as HashMap; +use futures::future::BoxFuture; + +use phoenix_api::rpc::{RpcId, StatusCode, TransportStatus}; +use phoenix_api_policy_metrics::control_plane; + +use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::MetricsConfig; + +pub(crate) struct MetricsEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + pub(crate) num_succ: u32, + pub(crate) num_rej: u32, + + pub(crate) config: MetricsConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for MetricsEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "MetricsEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + // Update config + self.config = MetricsConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(MetricsEngine, node); + +impl Decompose for MetricsEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + + let mut collections = ResourceCollection::with_capacity(2); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert("num_rej".to_string(), Box::new(engine.num_rej)); + collections.insert("num_succ".to_string(), Box::new(engine.num_succ)); + (collections, engine.node) + } +} + +impl MetricsEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let num_succ = *local + .remove("num_succ") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let num_rej = *local + .remove("num_rej") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let engine = MetricsEngine { + node, + indicator: Default::default(), + num_succ, + num_rej, + config, + }; + Ok(engine) + } +} + +impl MetricsEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +impl MetricsEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + self.tx_outputs()[0].send(msg)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + // forward all rx msgs + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + match m { + EngineRxMessage::RpcMessage(msg) => { + let meta = unsafe { &*msg.meta.as_ptr() }; + if meta.status_code == StatusCode::Success { + self.num_succ += 1; + } else { + self.num_rej += 1; + } + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => { + self.rx_outputs()[0].send(m)?; + } + }; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/metrics/src/lib.rs b/experimental/mrpc/plugin/policy/metrics/src/lib.rs new file mode 100644 index 00000000..0f40afdf --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics/src/lib.rs @@ -0,0 +1,34 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] + +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::MetricsConfig; +use crate::module::MetricsAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = MetricsConfig::new(config_string)?; + let addon = MetricsAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/metrics/src/module.rs b/experimental/mrpc/plugin/policy/metrics/src/module.rs new file mode 100644 index 00000000..9aaef7a2 --- /dev/null +++ b/experimental/mrpc/plugin/policy/metrics/src/module.rs @@ -0,0 +1,102 @@ +use anyhow::{bail, Result}; +use fnv::FnvHashMap as HashMap; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::MetricsEngine; +use crate::config::MetricsConfig; + +pub(crate) struct MetricsEngineBuilder { + node: DataPathNode, + config: MetricsConfig, +} + +impl MetricsEngineBuilder { + fn new(node: DataPathNode, config: MetricsConfig) -> Self { + MetricsEngineBuilder { node, config } + } + + fn build(self) -> Result { + Ok(MetricsEngine { + node: self.node, + indicator: Default::default(), + num_succ: 0, + num_rej: 0, + config: self.config, + }) + } +} + +pub struct MetricsAddon { + config: MetricsConfig, +} + +impl MetricsAddon { + pub const METRICS_ENGINE: EngineType = EngineType("MetricsEngine"); + pub const ENGINES: &'static [EngineType] = &[MetricsAddon::METRICS_ENGINE]; +} + +impl MetricsAddon { + pub fn new(config: MetricsConfig) -> Self { + MetricsAddon { config } + } +} + +impl PhoenixAddon for MetricsAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + MetricsAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != MetricsAddon::METRICS_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = MetricsEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != MetricsAddon::METRICS_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = MetricsEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation-server/Cargo.toml b/experimental/mrpc/plugin/policy/mutation-server/Cargo.toml new file mode 100644 index 00000000..a6552545 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation-server/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "phoenix-mutation-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + + +[dependencies] +phoenix-api-policy-mutation-server.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true + +phoenix_common.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +fnv.workspace = true diff --git a/experimental/mrpc/plugin/policy/mutation-server/src/config.rs b/experimental/mrpc/plugin/policy/mutation-server/src/config.rs new file mode 100644 index 00000000..8928ba65 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation-server/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MutationServerConfig {} + +impl MutationServerConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation-server/src/engine.rs b/experimental/mrpc/plugin/policy/mutation-server/src/engine.rs new file mode 100644 index 00000000..97c6bbdb --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation-server/src/engine.rs @@ -0,0 +1,196 @@ +//! This engine can only be placed at the sender side for now. +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; + +use anyhow::{anyhow, Result}; +use fnv::FnvHashMap as HashMap; +use futures::future::BoxFuture; + +use phoenix_api::rpc::{RpcId, TransportStatus}; +use phoenix_api_policy_mutation_server::control_plane; + +use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::datapath::RpcMessageRx; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::MutationServerConfig; + +pub(crate) struct MutationServerEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + pub(crate) config: MutationServerConfig, + + pub(crate) target: String, +} + +pub mod hello { + // The string specified here must match the proto package name + include!("rpc_hello.rs"); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for MutationServerEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "MutationServerEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + // Update config + self.config = MutationServerConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(MutationServerEngine, node); + +impl Decompose for MutationServerEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + + let mut collections = ResourceCollection::with_capacity(2); + collections.insert("target".to_string(), Box::new(engine.target)); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl MutationServerEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let target = *local + .remove("target") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let engine = MutationServerEngine { + node, + indicator: Default::default(), + target, + config, + }; + Ok(engine) + } +} + +impl MutationServerEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageRx) -> &mut hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_mut().unwrap() }; + return req; +} + +impl MutationServerEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + self.tx_outputs()[0].send(msg)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + match m { + EngineRxMessage::Ack(rpc_id, _status) => { + self.rx_outputs()[0].send(m)?; + } + EngineRxMessage::RpcMessage(msg) => { + let mut req = materialize_nocopy(&msg); + for i in 0..req.name.len() { + req.name[i] = 'a' as u8; + } + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + EngineRxMessage::RecvError(_, _) => { + self.rx_outputs()[0].send(m)?; + } + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation-server/src/lib.rs b/experimental/mrpc/plugin/policy/mutation-server/src/lib.rs new file mode 100644 index 00000000..6705cf79 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation-server/src/lib.rs @@ -0,0 +1,34 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] + +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::MutationServerConfig; +use crate::module::MutationServerAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = MutationServerConfig::new(config_string)?; + let addon = MutationServerAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/mutation-server/src/module.rs b/experimental/mrpc/plugin/policy/mutation-server/src/module.rs new file mode 100644 index 00000000..9f60e8aa --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation-server/src/module.rs @@ -0,0 +1,101 @@ +use anyhow::{bail, Result}; +use fnv::FnvHashMap as HashMap; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::MutationServerEngine; +use crate::config::MutationServerConfig; + +pub(crate) struct MutationServerEngineBuilder { + node: DataPathNode, + config: MutationServerConfig, +} + +impl MutationServerEngineBuilder { + fn new(node: DataPathNode, config: MutationServerConfig) -> Self { + MutationServerEngineBuilder { node, config } + } + + fn build(self) -> Result { + Ok(MutationServerEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + target: "Banana".to_string(), + }) + } +} + +pub struct MutationServerAddon { + config: MutationServerConfig, +} + +impl MutationServerAddon { + pub const MUTATION_ENGINE: EngineType = EngineType("MutationServerEngine"); + pub const ENGINES: &'static [EngineType] = &[MutationServerAddon::MUTATION_ENGINE]; +} + +impl MutationServerAddon { + pub fn new(config: MutationServerConfig) -> Self { + MutationServerAddon { config } + } +} + +impl PhoenixAddon for MutationServerAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + MutationServerAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != MutationServerAddon::MUTATION_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = MutationServerEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != MutationServerAddon::MUTATION_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = MutationServerEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation-server/src/rpc_hello.rs b/experimental/mrpc/plugin/policy/mutation-server/src/rpc_hello.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation-server/src/rpc_hello.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/mutation/Cargo.toml b/experimental/mrpc/plugin/policy/mutation/Cargo.toml new file mode 100644 index 00000000..da4953c0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "phoenix-mutation" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + + +[dependencies] +phoenix-api-policy-mutation.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true + +phoenix_common.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +fnv.workspace = true diff --git a/experimental/mrpc/plugin/policy/mutation/src/config.rs b/experimental/mrpc/plugin/policy/mutation/src/config.rs new file mode 100644 index 00000000..ab1d2d5f --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation/src/config.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MutationConfig {} + +impl MutationConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation/src/engine.rs b/experimental/mrpc/plugin/policy/mutation/src/engine.rs new file mode 100644 index 00000000..ffb4d958 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation/src/engine.rs @@ -0,0 +1,195 @@ +//! This engine can only be placed at the sender side for now. +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; + +use anyhow::{anyhow, Result}; +use fnv::FnvHashMap as HashMap; +use futures::future::BoxFuture; + +use phoenix_api::rpc::{RpcId, TransportStatus}; +use phoenix_api_policy_mutation::control_plane; + +use phoenix_common::engine::datapath::message::{EngineRxMessage, EngineTxMessage, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::MutationConfig; + +pub(crate) struct MutationEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + pub(crate) config: MutationConfig, + + pub(crate) target: String, +} + +pub mod hello { + // The string specified here must match the proto package name + include!("rpc_hello.rs"); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for MutationEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "MutationEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig() => { + // Update config + self.config = MutationConfig {}; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(MutationEngine, node); + +impl Decompose for MutationEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + + let mut collections = ResourceCollection::with_capacity(2); + collections.insert("target".to_string(), Box::new(engine.target)); + collections.insert("config".to_string(), Box::new(engine.config)); + (collections, engine.node) + } +} + +impl MutationEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let target = *local + .remove("target") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let engine = MutationEngine { + node, + indicator: Default::default(), + target, + config, + }; + Ok(engine) + } +} + +impl MutationEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +/// Copy the RPC request to a private heap and returns the request. +#[inline] +fn materialize(msg: &RpcMessageTx) -> Box { + let req_ptr = Unique::new(msg.addr_backend as *mut hello::HelloRequest).unwrap(); + let req = unsafe { req_ptr.as_ref() }; + // returns a private_req + Box::new(req.clone()) +} + +impl MutationEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(msg) => { + let mut req_ptr = + Unique::new(msg.addr_backend as *mut hello::HelloRequest).unwrap(); + let req = unsafe { req_ptr.as_mut() }; + for i in 0..req.name.len() { + req.name[i] = 'a' as u8; + } + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(msg))?; + } + m => self.tx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + // forward all rx msgs + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + self.rx_outputs()[0].send(m)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation/src/lib.rs b/experimental/mrpc/plugin/policy/mutation/src/lib.rs new file mode 100644 index 00000000..04a1a081 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation/src/lib.rs @@ -0,0 +1,34 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] + +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::MutationConfig; +use crate::module::MutationAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = MutationConfig::new(config_string)?; + let addon = MutationAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/mutation/src/module.rs b/experimental/mrpc/plugin/policy/mutation/src/module.rs new file mode 100644 index 00000000..f7b47bca --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation/src/module.rs @@ -0,0 +1,101 @@ +use anyhow::{bail, Result}; +use fnv::FnvHashMap as HashMap; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::MutationEngine; +use crate::config::MutationConfig; + +pub(crate) struct MutationEngineBuilder { + node: DataPathNode, + config: MutationConfig, +} + +impl MutationEngineBuilder { + fn new(node: DataPathNode, config: MutationConfig) -> Self { + MutationEngineBuilder { node, config } + } + + fn build(self) -> Result { + Ok(MutationEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + target: "Banana".to_string(), + }) + } +} + +pub struct MutationAddon { + config: MutationConfig, +} + +impl MutationAddon { + pub const MUTATION_ENGINE: EngineType = EngineType("MutationEngine"); + pub const ENGINES: &'static [EngineType] = &[MutationAddon::MUTATION_ENGINE]; +} + +impl MutationAddon { + pub fn new(config: MutationConfig) -> Self { + MutationAddon { config } + } +} + +impl PhoenixAddon for MutationAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + MutationAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != MutationAddon::MUTATION_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = MutationEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != MutationAddon::MUTATION_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = MutationEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/mutation/src/rpc_hello.rs b/experimental/mrpc/plugin/policy/mutation/src/rpc_hello.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/mutation/src/rpc_hello.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop-server/Cargo.toml b/experimental/mrpc/plugin/policy/ratelimit-drop-server/Cargo.toml new file mode 100644 index 00000000..0f2a1eaa --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop-server/Cargo.toml @@ -0,0 +1,28 @@ + +[package] +name = "phoenix-ratelimit-drop-server" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-ratelimit-drop-server.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +chrono.workspace = true +itertools.workspace = true +rand.workspace = true diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/config.rs b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/config.rs new file mode 100644 index 00000000..086114ea --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/config.rs @@ -0,0 +1,48 @@ +use chrono::{Datelike, Timelike, Utc}; +use phoenix_common::log; +use serde::{Deserialize, Serialize}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] + +pub struct RateLimitDropServerConfig { + pub requests_per_sec: u64, + pub bucket_size: u64, +} + +impl Default for RateLimitDropServerConfig { + fn default() -> Self { + RateLimitDropServerConfig { + requests_per_sec: 100000, + bucket_size: 100000, + } + } +} + +impl RateLimitDropServerConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} +pub fn create_log_file() -> std::fs::File { + std::fs::create_dir_all("/tmp/phoenix/log").expect("mkdir failed"); + let now = Utc::now(); + let date_string = format!( + "{}-{}-{}-{}-{}-{}", + now.year(), + now.month(), + now.day(), + now.hour(), + now.minute(), + now.second() + ); + let file_name = format!("/tmp/phoenix/log/logging_engine_{}.log", date_string); + ///log::info!("create log file {}", file_name); + let log_file = std::fs::File::create(file_name).expect("create file failed"); + log_file +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/engine.rs b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/engine.rs new file mode 100644 index 00000000..cb5ecea9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/engine.rs @@ -0,0 +1,276 @@ +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use minstant::Instant; +use phoenix_api::rpc::{RpcId, StatusCode, TransportStatus}; +use std::fmt; +use std::fs::File; +use std::io::Write; +use std::num::NonZeroU32; +use std::os::unix::ucred::UCred; +use std::pin::Pin; +use std::ptr::Unique; + +use phoenix_api_policy_ratelimit_drop_server::control_plane; +use phoenix_common::engine::datapath::message::{ + EngineRxMessage, EngineTxMessage, RpcMessageGeneral, +}; +use phoenix_common::engine::datapath::meta_pool::MetaBufferPool; + +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; + +use phoenix_common::engine::datapath::{RpcMessageRx, RpcMessageTx}; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; + +use super::DatapathError; +use crate::config::{create_log_file, RateLimitDropServerConfig}; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub(crate) struct RateLimitDropServerEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + // A set of func_ids to apply the rate limit. + // TODO(cjr): maybe put this filter in a separate engine like FilterEngine/ClassiferEngine. + // pub(crate) filter: FnvHashSet, + // Number of tokens to add for each seconds. + pub(crate) config: RateLimitDropServerConfig, + pub(crate) meta_buf_pool: MetaBufferPool, + + // The most recent timestamp we add the token to the bucket. + pub(crate) last_ts: Instant, + // The number of available tokens in the token bucket algorithm. + pub(crate) num_tokens: f64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for RateLimitDropServerEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "RateLimitDropServerEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig(requests_per_sec, bucket_size) => { + self.config = RateLimitDropServerConfig { + requests_per_sec, + bucket_size, + }; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(RateLimitDropServerEngine, node); + +impl Decompose for RateLimitDropServerEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() || !self.rx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert("meta_buf_pool".to_string(), Box::new(engine.meta_buf_pool)); + collections.insert("last_ts".to_string(), Box::new(engine.last_ts)); + collections.insert("num_tokens".to_string(), Box::new(engine.num_tokens)); + (collections, engine.node) + } +} + +impl RateLimitDropServerEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let meta_buf_pool = *local + .remove("meta_buf_pool") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let last_ts: Instant = *local + .remove("last_ts") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let num_tokens = *local + .remove("num_tokens") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + + let engine = RateLimitDropServerEngine { + node, + indicator: Default::default(), + config, + last_ts, + num_tokens, + meta_buf_pool, + }; + Ok(engine) + } +} + +impl RateLimitDropServerEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + self.indicator.set_nwork(work); + future::yield_now().await; + } + } +} + +fn current_timestamp() -> Instant { + Instant::now() +} +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +/// Copy the RPC request to a private heap and returns the request. +#[inline] +fn materialize_rx(msg: &RpcMessageRx) -> Box { + let req_ptr = Unique::new(msg.addr_backend as *mut hello::HelloRequest).unwrap(); + let req = unsafe { req_ptr.as_ref() }; + // returns a private_req + Box::new(req.clone()) +} + +impl RateLimitDropServerEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + self.tx_outputs()[0].send(msg)?; + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + + match self.rx_inputs()[0].try_recv() { + Ok(m) => { + match m { + EngineRxMessage::Ack(rpc_id, status) => { + if let Ok(()) = self.meta_buf_pool.release(rpc_id) { + } else { + self.rx_outputs()[0].send(m)?; + } + } + EngineRxMessage::RpcMessage(msg) => { + self.num_tokens = self.num_tokens + + (current_timestamp() - self.last_ts).as_secs_f64() + * self.config.requests_per_sec as f64; + self.last_ts = current_timestamp(); + if self.num_tokens < 1.0 { + // We need to copy meta, add it to meta_buf_pool, and send it as the tx msg + // Is there better way to do this and avoid unsafe? + let mut meta = unsafe { msg.meta.as_ref().clone() }; + meta.status_code = StatusCode::AccessDenied; + let mut meta_ptr = self + .meta_buf_pool + .obtain(RpcId(meta.conn_id, meta.call_id)) + .expect("meta_buf_pool is full"); + unsafe { + meta_ptr.as_meta_ptr().write(meta); + meta_ptr.0.as_mut().num_sge = 0; + meta_ptr.0.as_mut().value_len = 0; + } + let rpc_msg = RpcMessageTx { + meta_buf_ptr: meta_ptr, + addr_backend: 0, + }; + let new_msg = EngineTxMessage::RpcMessage(rpc_msg); + self.tx_outputs()[0] + .send(new_msg) + .expect("send new message error"); + let msg_call_ids = + [meta.call_id, meta.call_id, meta.call_id, meta.call_id]; + self.tx_outputs()[0].send(EngineTxMessage::ReclaimRecvBuf( + meta.conn_id, + msg_call_ids, + ))?; + } else { + self.num_tokens = self.num_tokens - 1.0; + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + } + EngineRxMessage::RecvError(_, _) => { + self.rx_outputs()[0].send(m)?; + } + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/lib.rs b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/lib.rs new file mode 100644 index 00000000..15fd5311 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/lib.rs @@ -0,0 +1,37 @@ +#![feature(peer_credentials_unix_socket)] +#![feature(ptr_internals)] +#![feature(strict_provenance)] +use thiserror::Error; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::RateLimitDropServerConfig; +use crate::module::RateLimitDropServerAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = RateLimitDropServerConfig::new(config_string)?; + let addon = RateLimitDropServerAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/module.rs b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/module.rs new file mode 100644 index 00000000..244bdff9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/module.rs @@ -0,0 +1,109 @@ +use anyhow::{bail, Result}; +use minstant::Instant; +use nix::unistd::Pid; + +use super::engine::RateLimitDropServerEngine; +use crate::config::{create_log_file, RateLimitDropServerConfig}; +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::meta_pool::{MetaBufferPool, META_BUFFER_SIZE}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use chrono::prelude::*; +use itertools::iproduct; +use rand::Rng; + +pub(crate) struct RateLimitDropServerEngineBuilder { + node: DataPathNode, + config: RateLimitDropServerConfig, +} + +impl RateLimitDropServerEngineBuilder { + fn new(node: DataPathNode, config: RateLimitDropServerConfig) -> Self { + RateLimitDropServerEngineBuilder { node, config } + } + // TODO! LogFile + fn build(self) -> Result { + let META_BUFFER_POOL_CAP = 200; + Ok(RateLimitDropServerEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + meta_buf_pool: MetaBufferPool::new(META_BUFFER_POOL_CAP), + last_ts: Instant::now(), + num_tokens: self.config.bucket_size as _, + }) + } +} + +pub struct RateLimitDropServerAddon { + config: RateLimitDropServerConfig, +} + +impl RateLimitDropServerAddon { + pub const RATELIMIT_DROP_SERVER_ENGINE: EngineType = EngineType("RateLimitDropServerEngine"); + pub const ENGINES: &'static [EngineType] = + &[RateLimitDropServerAddon::RATELIMIT_DROP_SERVER_ENGINE]; +} + +impl RateLimitDropServerAddon { + pub fn new(config: RateLimitDropServerConfig) -> Self { + RateLimitDropServerAddon { config } + } +} + +impl PhoenixAddon for RateLimitDropServerAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + RateLimitDropServerAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != RateLimitDropServerAddon::RATELIMIT_DROP_SERVER_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = RateLimitDropServerEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != RateLimitDropServerAddon::RATELIMIT_DROP_SERVER_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = RateLimitDropServerEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/proto.rs b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop-server/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop/Cargo.toml b/experimental/mrpc/plugin/policy/ratelimit-drop/Cargo.toml new file mode 100644 index 00000000..4b592492 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "phoenix-ratelimit-drop" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +phoenix_common.workspace = true +phoenix-api-policy-ratelimit-drop.workspace = true + +futures.workspace = true +minstant.workspace = true +thiserror.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +nix.workspace = true +toml = { workspace = true, features = ["preserve_order"] } +bincode.workspace = true +mrpc-marshal.workspace = true +mrpc-derive.workspace = true +shm.workspace = true +phoenix-api = { workspace = true, features = ["mrpc"] } +chrono.workspace = true +itertools.workspace = true diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop/src/config.rs b/experimental/mrpc/plugin/policy/ratelimit-drop/src/config.rs new file mode 100644 index 00000000..7f4af438 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop/src/config.rs @@ -0,0 +1,24 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RateLimitDropConfig { + pub requests_per_sec: u64, + pub bucket_size: u64, +} + +impl Default for RateLimitDropConfig { + fn default() -> Self { + RateLimitDropConfig { + requests_per_sec: 100000, + bucket_size: 100000, + } + } +} + +impl RateLimitDropConfig { + pub fn new(config: Option<&str>) -> anyhow::Result { + let config = toml::from_str(config.unwrap_or(""))?; + Ok(config) + } +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop/src/engine.rs b/experimental/mrpc/plugin/policy/ratelimit-drop/src/engine.rs new file mode 100644 index 00000000..3728d586 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop/src/engine.rs @@ -0,0 +1,241 @@ +use std::collections::VecDeque; +use std::os::unix::ucred::UCred; +use std::pin::Pin; + +use super::DatapathError; +use crate::config::RateLimitDropConfig; +use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; +use minstant::Instant; +use phoenix_api::rpc::{RpcId, TransportStatus}; +use phoenix_api_policy_ratelimit_drop::control_plane; +use phoenix_common::engine::datapath::message::{EngineTxMessage, RpcMessageGeneral, RpcMessageTx}; +use phoenix_common::engine::datapath::node::DataPathNode; +use phoenix_common::engine::datapath::EngineRxMessage; +use phoenix_common::engine::{future, Decompose, Engine, EngineResult, Indicator, Vertex}; +use phoenix_common::envelop::ResourceDowncast; +use phoenix_common::impl_vertex_for_engine; +use phoenix_common::log; +use phoenix_common::module::Version; +use phoenix_common::storage::{ResourceCollection, SharedStorage}; +use std::num::NonZeroU32; + +pub mod hello { + include!("proto.rs"); +} + +fn hello_request_name_readonly(req: &hello::HelloRequest) -> String { + let buf = &req.name as &[u8]; + String::from_utf8_lossy(buf).to_string().clone() +} + +pub(crate) struct RateLimitDropEngine { + pub(crate) node: DataPathNode, + + pub(crate) indicator: Indicator, + + // A set of func_ids to apply the rate limit. + // TODO(cjr): maybe put this filter in a separate engine like FilterEngine/ClassiferEngine. + // pub(crate) filter: FnvHashSet, + // Number of tokens to add for each seconds. + pub(crate) config: RateLimitDropConfig, + // The most recent timestamp we add the token to the bucket. + pub(crate) last_ts: Instant, + // The number of available tokens in the token bucket algorithm. + pub(crate) num_tokens: f64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Progress(usize), + Disconnected, +} + +use Status::Progress; + +impl Engine for RateLimitDropEngine { + fn activate<'a>(self: Pin<&'a mut Self>) -> BoxFuture<'a, EngineResult> { + Box::pin(async move { self.get_mut().mainloop().await }) + } + + fn description(self: Pin<&Self>) -> String { + "RateLimitDropEngine".to_owned() + } + + #[inline] + fn tracker(self: Pin<&mut Self>) -> &mut Indicator { + &mut self.get_mut().indicator + } + + fn handle_request(&mut self, request: Vec, _cred: UCred) -> Result<()> { + let request: control_plane::Request = bincode::deserialize(&request[..])?; + + match request { + control_plane::Request::NewConfig(requests_per_sec, bucket_size) => { + self.config = RateLimitDropConfig { + requests_per_sec, + bucket_size, + }; + } + } + Ok(()) + } +} + +impl_vertex_for_engine!(RateLimitDropEngine, node); + +impl Decompose for RateLimitDropEngine { + fn flush(&mut self) -> Result { + let mut work = 0; + while !self.tx_inputs()[0].is_empty() { + if let Progress(n) = self.check_input_queue()? { + work += n; + } + } + + Ok(work) + } + + fn decompose( + self: Box, + _shared: &mut SharedStorage, + _global: &mut ResourceCollection, + ) -> (ResourceCollection, DataPathNode) { + let engine = *self; + + let mut collections = ResourceCollection::with_capacity(4); + collections.insert("config".to_string(), Box::new(engine.config)); + collections.insert("last_ts".to_string(), Box::new(engine.last_ts)); + collections.insert("num_tokens".to_string(), Box::new(engine.num_tokens)); + (collections, engine.node) + } +} + +impl RateLimitDropEngine { + pub(crate) fn restore( + mut local: ResourceCollection, + node: DataPathNode, + _prev_version: Version, + ) -> Result { + let config = *local + .remove("config") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let last_ts: Instant = *local + .remove("last_ts") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + let num_tokens = *local + .remove("num_tokens") + .unwrap() + .downcast::() + .map_err(|x| anyhow!("fail to downcast, type_name={:?}", x.type_name()))?; + + let engine = RateLimitDropEngine { + node, + indicator: Default::default(), + config, + last_ts, + num_tokens, + }; + Ok(engine) + } +} + +impl RateLimitDropEngine { + async fn mainloop(&mut self) -> EngineResult { + loop { + let mut work = 0; + // check input queue, ~100ns + loop { + match self.check_input_queue()? { + Progress(0) => break, + Progress(n) => work += n, + Status::Disconnected => return Ok(()), + } + } + + // If there's pending receives, there will always be future work to do. + self.indicator.set_nwork(work); + + future::yield_now().await; + } + } +} + +fn current_timestamp() -> Instant { + Instant::now() +} + +#[inline] +fn materialize_nocopy(msg: &RpcMessageTx) -> &hello::HelloRequest { + let req_ptr = msg.addr_backend as *mut hello::HelloRequest; + let req = unsafe { req_ptr.as_ref().unwrap() }; + return req; +} + +impl RateLimitDropEngine { + fn check_input_queue(&mut self) -> Result { + use phoenix_common::engine::datapath::TryRecvError; + + match self.tx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineTxMessage::RpcMessage(req) => { + self.num_tokens = self.num_tokens + + (current_timestamp() - self.last_ts).as_secs_f64() + * self.config.requests_per_sec as f64; + self.last_ts = current_timestamp(); + let rpc_message = materialize_nocopy(&req); + let conn_id = unsafe { &*req.meta_buf_ptr.as_meta_ptr() }.conn_id; + let call_id = unsafe { &*req.meta_buf_ptr.as_meta_ptr() }.call_id; + let rpc_id = RpcId::new(conn_id, call_id); + if self.num_tokens > 1.0 { + let raw_ptr: *const hello::HelloRequest = rpc_message; + let new_msg = RpcMessageTx { + meta_buf_ptr: req.meta_buf_ptr.clone(), + addr_backend: req.addr_backend, + }; + self.tx_outputs()[0].send(EngineTxMessage::RpcMessage(new_msg))?; + } else { + self.num_tokens = self.num_tokens - 1.0; + let error = EngineRxMessage::Ack( + rpc_id, + TransportStatus::Error(unsafe { NonZeroU32::new_unchecked(403) }), + ); + self.rx_outputs()[0].send(error)?; + } + } + m => { + self.tx_outputs()[0].send(m)?; + } + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => return Ok(Status::Disconnected), + } + match self.rx_inputs()[0].try_recv() { + Ok(msg) => { + match msg { + EngineRxMessage::Ack(rpc_id, status) => { + // todo + self.rx_outputs()[0].send(EngineRxMessage::Ack(rpc_id, status))?; + } + EngineRxMessage::RpcMessage(msg) => { + self.rx_outputs()[0].send(EngineRxMessage::RpcMessage(msg))?; + } + m => self.rx_outputs()[0].send(m)?, + } + return Ok(Progress(1)); + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + return Ok(Status::Disconnected); + } + } + Ok(Progress(0)) + } +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop/src/lib.rs b/experimental/mrpc/plugin/policy/ratelimit-drop/src/lib.rs new file mode 100644 index 00000000..11be89f9 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop/src/lib.rs @@ -0,0 +1,32 @@ +#![feature(peer_credentials_unix_socket)] + +use thiserror::Error; + +pub use phoenix_common::{InitFnResult, PhoenixAddon}; + +pub mod config; +pub(crate) mod engine; +pub mod module; + +#[derive(Error, Debug)] +pub(crate) enum DatapathError { + #[error("Internal queue send error")] + InternalQueueSend, +} + +use phoenix_common::engine::datapath::SendError; +impl From> for DatapathError { + fn from(_other: SendError) -> Self { + DatapathError::InternalQueueSend + } +} + +use crate::config::RateLimitDropConfig; +use crate::module::RateLimitDropAddon; + +#[no_mangle] +pub fn init_addon(config_string: Option<&str>) -> InitFnResult> { + let config = RateLimitDropConfig::new(config_string)?; + let addon = RateLimitDropAddon::new(config); + Ok(Box::new(addon)) +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop/src/module.rs b/experimental/mrpc/plugin/policy/ratelimit-drop/src/module.rs new file mode 100644 index 00000000..156ee473 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop/src/module.rs @@ -0,0 +1,104 @@ +use std::collections::VecDeque; + +use anyhow::{bail, Result}; +use minstant::Instant; +use nix::unistd::Pid; + +use phoenix_common::addon::{PhoenixAddon, Version}; +use phoenix_common::engine::datapath::DataPathNode; +use phoenix_common::engine::{Engine, EngineType}; +use phoenix_common::storage::ResourceCollection; + +use super::engine::RateLimitDropEngine; +use crate::config::RateLimitDropConfig; + +pub(crate) struct RateLimitDropEngineBuilder { + node: DataPathNode, + config: RateLimitDropConfig, +} + +impl RateLimitDropEngineBuilder { + fn new(node: DataPathNode, config: RateLimitDropConfig) -> Self { + RateLimitDropEngineBuilder { node, config } + } + + fn build(self) -> Result { + Ok(RateLimitDropEngine { + node: self.node, + indicator: Default::default(), + config: self.config, + last_ts: Instant::now(), + num_tokens: self.config.bucket_size as _, + }) + } +} + +pub struct RateLimitDropAddon { + config: RateLimitDropConfig, +} + +impl RateLimitDropAddon { + pub const RATE_LIMIT_DROP_ENGINE: EngineType = EngineType("RateLimitDropEngine"); + pub const ENGINES: &'static [EngineType] = &[RateLimitDropAddon::RATE_LIMIT_DROP_ENGINE]; +} + +impl RateLimitDropAddon { + pub fn new(config: RateLimitDropConfig) -> Self { + RateLimitDropAddon { config } + } +} + +impl PhoenixAddon for RateLimitDropAddon { + fn check_compatibility(&self, _prev: Option<&Version>) -> bool { + true + } + + fn decompose(self: Box) -> ResourceCollection { + let addon = *self; + let mut collections = ResourceCollection::new(); + collections.insert("config".to_string(), Box::new(addon.config)); + collections + } + + #[inline] + fn migrate(&mut self, _prev_addon: Box) {} + + fn engines(&self) -> &[EngineType] { + RateLimitDropAddon::ENGINES + } + + fn update_config(&mut self, config: &str) -> Result<()> { + self.config = toml::from_str(config)?; + Ok(()) + } + + fn create_engine( + &mut self, + ty: EngineType, + _pid: Pid, + node: DataPathNode, + ) -> Result> { + if ty != RateLimitDropAddon::RATE_LIMIT_DROP_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let builder = RateLimitDropEngineBuilder::new(node, self.config); + let engine = builder.build()?; + Ok(Box::new(engine)) + } + + fn restore_engine( + &mut self, + ty: EngineType, + local: ResourceCollection, + node: DataPathNode, + prev_version: Version, + ) -> Result> { + if ty != RateLimitDropAddon::RATE_LIMIT_DROP_ENGINE { + bail!("invalid engine type {:?}", ty) + } + + let engine = RateLimitDropEngine::restore(local, node, prev_version)?; + Ok(Box::new(engine)) + } +} diff --git a/experimental/mrpc/plugin/policy/ratelimit-drop/src/proto.rs b/experimental/mrpc/plugin/policy/ratelimit-drop/src/proto.rs new file mode 100644 index 00000000..83ee8ab0 --- /dev/null +++ b/experimental/mrpc/plugin/policy/ratelimit-drop/src/proto.rs @@ -0,0 +1,29 @@ +/// The request message containing the user's name. +#[repr(C)] +#[derive(Debug, Clone, ::mrpc_derive::Message)] +pub struct HelloRequest { + #[prost(bytes = "vec", tag = "1")] + pub name: ::mrpc_marshal::shadow::Vec, +} +/// The response message containing the greetings +#[repr(C)] +#[derive(Debug, ::mrpc_derive::Message)] +pub struct HelloReply { + #[prost(bytes = "vec", tag = "1")] + pub message: ::mrpc_marshal::shadow::Vec, +} + +// /// The request message containing the user's name. +// #[repr(C)] +// #[derive(Debug, Clone, ::mrpc_derive::Message)] +// pub struct HelloRequest { +// #[prost(bytes = "vec", tag = "1")] +// pub name: ::mrpc::alloc::Vec, +// } +// /// The response message containing the greetings +// #[repr(C)] +// #[derive(Debug, ::mrpc_derive::Message)] +// pub struct HelloReply { +// #[prost(bytes = "vec", tag = "1")] +// pub message: ::mrpc::alloc::Vec, +// } diff --git a/phoenix.toml b/phoenix.toml index be489c0d..032a4340 100644 --- a/phoenix.toml +++ b/phoenix.toml @@ -55,3 +55,111 @@ lib_path = "plugins/libphoenix_salloc.rlib" # requests_per_sec = 1000 # bucket_size = 1000 # ''' +[[modules]] +name = "Mrpc" +lib_path = "plugins/libphoenix_mrpc.rlib" +config_string = ''' +prefix = "/tmp/phoenix" +engine_basename = "mrpc-engine" +build_cache = "/tmp/phoenix/build-cache" +transport = "Rdma" +nic_index = 0 +''' + +[[modules]] +name = "RpcAdapter" +lib_path = "plugins/libphoenix_rpc_adapter.rlib" +config_string = ''' +enable_scheduler = false +''' + + +[[modules]] +name = "TcpRpcAdapter" +lib_path = "plugins/libphoenix_tcp_rpc_adapter.rlib" + + +[[addons]] +name = "RateLimit" +lib_path = "plugins/libphoenix_ratelimit.rlib" +config_string = ''' +requests_per_sec = 1000 +bucket_size = 1000 +''' + +[[addons]] +name = "Qos" +lib_path = "plugins/libphoenix_qos.rlib" +config_string = ''' +latency_budget_microsecs = 10 +''' + +[[addons]] +name = "HotelAcl" +lib_path = "plugins/libphoenix_hotel_acl.rlib" +config_string = ''' +''' + +[[addons]] +name = "Null" +lib_path = "plugins/libphoenix_null.rlib" +config_string = ''' +''' + +[[addons]] +name = "logging" +lib_path = "plugins/libphoenix_logging.rlib" +config_string = ''' +''' + +[[addons]] +name = "HelloAclReceiver" +lib_path = "plugins/libphoenix_hello_acl_receiver.rlib" +config_string = ''' +''' + +[[addons]] +name = "HelloAclSender" +lib_path = "plugins/libphoenix_hello_acl_sender.rlib" +config_string = ''' +''' + +[[addons]] +name = "HelloAcl" +lib_path = "plugins/libphoenix_hello_acl.rlib" +config_string = ''' +''' + +[[addons]] +name = "NofileLogging" +lib_path = "plugins/libphoenix_nofile_logging.rlib" +config_string = ''' +''' + +[[addons]] +name = "Fault" +lib_path = "plugins/libphoenix_fault.rlib" +config_string = ''' +''' + +[[addons]] +name = "Fault2" +lib_path = "plugins/libphoenix_fault2.rlib" +config_string = ''' +''' + +[[addons]] +name = "Delay" +lib_path = "plugins/libphoenix_delay.rlib" +config_string = ''' +delay_probability = 0.2 +delay_ms = 100 +''' + +[[addons]] +name = "RateLimitDrop" +lib_path = "plugins/libphoenix_ratelimit_drop.rlib" +config_string = ''' +requests_per_sec = 4 +bucket_size = 1000 +''' diff --git a/src/rdma/src/ibv.rs b/src/rdma/src/ibv.rs index 6aa0ddba..70e246cc 100644 --- a/src/rdma/src/ibv.rs +++ b/src/rdma/src/ibv.rs @@ -190,21 +190,21 @@ impl<'devlist> Device<'devlist> { } } - /// Returns stable IB device index as it is assigned by the kernel - /// # Errors - /// - /// - `ENOTSUP`: Stable index is not supported - pub fn index(&self) -> io::Result { - let idx = unsafe { ffi::ibv_get_device_index(*self.0) }; - if idx == -1 { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "device index not known", - )) - } else { - Ok(idx) - } - } + // /// Returns stable IB device index as it is assigned by the kernel + // /// # Errors + // /// + // /// - `ENOTSUP`: Stable index is not supported + // pub fn index(&self) -> io::Result { + // let idx = unsafe { ffi::ibv_get_device_index(*self.0) }; + // if idx == -1 { + // Err(io::Error::new( + // io::ErrorKind::Unsupported, + // "device index not known", + // )) + // } else { + // Ok(idx) + // } + // } } /// An RDMA context bound to a device.