Split preloading and client defs (#5493)

microsoft · Aug 2, 2023 · edda3bb · edda3bb
1 parent a6fdd26
commit edda3bb
Show file tree

Hide file tree

Showing 8 changed files with 276 additions and 148 deletions.
diff --git a/.cmake-format.py b/.cmake-format.py
@@ -40,6 +40,17 @@
                 "ADDITIONAL_ARGS": "*",
             },
         },
+        "add_piccolo_test": {
+            "kwargs": {
+                "NAME": "*",
+                "PYTHON_SCRIPT": "*",
+                "CONSTITUTION": "*",
+                "CLIENT_BIN": "*",
+                "VERIFICATION_FILE": "*",
+                "LABEL": "*",
+                "ADDITIONAL_ARGS": "*",
+            },
+        },
         "add_picobench": {
             "kwargs": {
                 "SRCS": "*",

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1438,29 +1438,29 @@ if(BUILD_TESTS)
                       --max-writes-ahead 1000 --repetitions 10000
     )
 
-    add_perf_test(
+    add_piccolo_test(
       NAME pi_basic
       PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/basicperf.py
       CLIENT_BIN ./submit
-      ADDITIONAL_ARGS --package "samples/apps/basic/libbasic" --repetitions
-                      100000
+      ADDITIONAL_ARGS --package "samples/apps/basic/libbasic" --client-def
+                      "1,write,100000,primary"
     )
 
-    add_perf_test(
+    add_piccolo_test(
       NAME pi_basic_js
       PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/basicperf.py
       CLIENT_BIN ./submit
       ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/basic/js
-                      --repetitions 100000
+                      --client-def "1,write,100000,primary"
     )
 
     if(WORKER_THREADS)
-      add_perf_test(
+      add_piccolo_test(
         NAME pi_basic_mt
         PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/basicperf.py
         CLIENT_BIN ./submit
-        ADDITIONAL_ARGS --package "samples/apps/basic/libbasic" --repetitions
-                        200000 --num-localhost-clients ${WORKER_THREADS}
+        ADDITIONAL_ARGS --package "samples/apps/basic/libbasic" --client-def
+                        "${WORKER_THREADS},write,100000,any"
       )
     endif()
 

diff --git a/cmake/common.cmake b/cmake/common.cmake
@@ -266,6 +266,85 @@ function(add_perf_test)
   )
 endfunction()
 
+# Helper for building end-to-end perf tests using the python infrastucture
+function(add_piccolo_test)
+
+  cmake_parse_arguments(
+    PARSE_ARGV 0 PARSED_ARGS ""
+    "NAME;PYTHON_SCRIPT;CONSTITUTION;CLIENT_BIN;VERIFICATION_FILE;LABEL"
+    "ADDITIONAL_ARGS"
+  )
+
+  if(NOT PARSED_ARGS_CONSTITUTION)
+    set(PARSED_ARGS_CONSTITUTION ${CCF_NETWORK_TEST_DEFAULT_CONSTITUTION})
+  endif()
+
+  set(TESTS_SUFFIX "")
+  set(ENCLAVE_TYPE "")
+  set(ENCLAVE_PLATFORM "${COMPILE_TARGET}")
+  if("sgx" STREQUAL COMPILE_TARGET)
+    set(TESTS_SUFFIX "${TESTS_SUFFIX}_sgx")
+    set(ENCLAVE_TYPE "release")
+  elseif("virtual" STREQUAL COMPILE_TARGET)
+    set(TESTS_SUFFIX "${TESTS_SUFFIX}_virtual")
+    set(ENCLAVE_TYPE "virtual")
+  endif()
+
+  set(TESTS_SUFFIX "${TESTS_SUFFIX}_cft")
+
+  set(TEST_NAME "${PARSED_ARGS_NAME}${TESTS_SUFFIX}")
+
+  set(LABEL_ARG "${TEST_NAME}^")
+
+  add_test(
+    NAME "${PARSED_ARGS_NAME}${TESTS_SUFFIX}"
+    COMMAND
+      ${PYTHON} ${PARSED_ARGS_PYTHON_SCRIPT} -b . -c ${PARSED_ARGS_CLIENT_BIN}
+      ${CCF_NETWORK_TEST_ARGS} ${PARSED_ARGS_CONSTITUTION} ${VERIFICATION_ARG}
+      --label ${LABEL_ARG} --snapshot-tx-interval 10000
+      ${PARSED_ARGS_ADDITIONAL_ARGS} -e ${ENCLAVE_TYPE} -t ${ENCLAVE_PLATFORM}
+      ${NODES}
+  )
+
+  # Make python test client framework importable
+  set_property(
+    TEST ${TEST_NAME}
+    APPEND
+    PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
+  )
+  if(DEFINED DEFAULT_ENCLAVE_TYPE)
+    set_property(
+      TEST ${TEST_NAME}
+      APPEND
+      PROPERTY ENVIRONMENT "DEFAULT_ENCLAVE_TYPE=${DEFAULT_ENCLAVE_TYPE}"
+    )
+  endif()
+  if(DEFINED DEFAULT_ENCLAVE_PLATFORM)
+    set_property(
+      TEST ${TEST_NAME}
+      APPEND
+      PROPERTY ENVIRONMENT
+               "DEFAULT_ENCLAVE_PLATFORM=${DEFAULT_ENCLAVE_PLATFORM}"
+    )
+  endif()
+  set_property(
+    TEST ${TEST_NAME}
+    APPEND
+    PROPERTY LABELS perf
+  )
+  set_property(
+    TEST ${TEST_NAME}
+    APPEND
+    PROPERTY LABELS cft
+  )
+  set_property(
+    TEST ${TEST_NAME}
+    APPEND
+    PROPERTY ENVIRONMENT
+             "TSAN_OPTIONS=suppressions=${CCF_DIR}/tsan_env_suppressions"
+  )
+endfunction()
+
 # Picobench wrapper
 function(add_picobench name)
   cmake_parse_arguments(

diff --git a/samples/apps/basic/basic.cpp b/samples/apps/basic/basic.cpp
@@ -86,6 +86,23 @@ namespace basicapp
       make_read_only_endpoint(
         "/records/{key}", HTTP_GET, get, {ccf::user_cert_auth_policy})
         .install();
+
+      auto post = [this](ccf::endpoints::EndpointContext& ctx) {
+        const nlohmann::json body =
+          nlohmann::json::parse(ctx.rpc_ctx->get_request_body());
+
+        const auto records = body.get<std::map<std::string, std::string>>();
+
+        auto records_handle = ctx.tx.template rw<RecordsMap>(PRIVATE_RECORDS);
+        for (const auto& [key, value] : records)
+        {
+          const std::vector<uint8_t> value_vec(value.begin(), value.end());
+          records_handle->put(key, value_vec);
+        }
+        ctx.rpc_ctx->set_response_status(HTTP_STATUS_NO_CONTENT);
+      };
+      make_endpoint("/records", HTTP_POST, post, {ccf::user_cert_auth_policy})
+        .install();
     }
   };
 }

diff --git a/samples/apps/basic/js/app.json b/samples/apps/basic/js/app.json
@@ -17,6 +17,16 @@
         "mode": "readwrite",
         "openapi": {}
       }
+    },
+    "/records": {
+      "post": {
+        "js_module": "basic.js",
+        "js_function": "post_records",
+        "forwarding_required": "always",
+        "authn_policies": ["user_cert"],
+        "mode": "readwrite",
+        "openapi": {}
+      }
     }
   }
 }
diff --git a/samples/apps/basic/js/src/basic.js b/samples/apps/basic/js/src/basic.js
@@ -32,3 +32,15 @@ export function get_record(request) {
     body: val,
   };
 }
+
+export function post_records(request) {
+  const records = request.body.json();
+
+  for (let key in records) {
+    records_table.set(ccf.strToBuf(key), ccf.strToBuf(records[key]));
+  }
+
+  return {
+    statusCode: 204,
+  };
+}