Build p4runtime protobufs

.github/workflows/pipeline.yml

@@ -39,6 +39,7 @@ jobs:
   #---------------------------------------------------------------------
   build_dpdk_p4cp:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
 
     steps:
       - name: Clone networking-recipe
@@ -51,13 +52,23 @@ jobs:
         run: |
           sudo apt install $PREREQS
 
+      - name: Set up Go environment
+        uses: actions/setup-go@v4.1.0
+        with:
+          go-version: 'stable'
+
+      - name: Install gRPC plugins
+        run: |
+          go version
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2
+
       - name: Install DPDK SDE
         uses: robinraju/release-downloader@v1.8
         with:
           repository: ${{ env.SDE_REPOSITORY }}
           tag: ${{ env.SDE_TAG }}
           fileName: ${{ env.SDE_FILENAME }}
-
       - run: |
           sudo tar -xzf $SDE_FILENAME -C /
           rm $SDE_FILENAME
@@ -68,7 +79,6 @@ jobs:
           repository: ${{ env.DEPS_REPOSITORY }}
           tag: ${{ env.DEPS_TAG }}
           fileName: ${{ env.DEPS_FILENAME }}
-
       - run: |
           sudo tar -xzf $DEPS_FILENAME -C /
           rm $DEPS_FILENAME
@@ -85,6 +95,7 @@ jobs:
   #---------------------------------------------------------------------
   krnlmon_unit_tests:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
 
     steps:
       - name: Clone networking-recipe
@@ -106,7 +117,6 @@ jobs:
           repository: ${{ env.SDE_REPOSITORY }}
           tag: ${{ env.SDE_TAG }}
           fileName: ${{ env.SDE_FILENAME }}
-
       - run: |
           sudo tar -xzf $SDE_FILENAME -C /
           rm $SDE_FILENAME
@@ -117,7 +127,6 @@ jobs:
           repository: ${{ env.DEPS_REPOSITORY }}
           tag: ${{ env.DEPS_TAG }}
           fileName: ${{ env.DEPS_FILENAME }}
-
       - run: |
           sudo tar -xzf $DEPS_FILENAME -C /
           rm $DEPS_FILENAME
@@ -134,7 +143,7 @@ jobs:
   # 3-py_bandit_check
   #---------------------------------------------------------------------
   py_bandit_check:
-     timeout-minutes: 10
+     timeout-minutes: 5
      runs-on: ubuntu-latest
      steps:
        - name: Clone networking-recipe
@@ -148,3 +157,64 @@ jobs:
            targets: |  # or a single string "."
             ./recipe/clients/p4rt-ctl/p4rt-ctl.in
            options: "-v"
+
+  #---------------------------------------------------------------------
+  # 4-build_p4runtime_protos
+  #---------------------------------------------------------------------
+  build_p4runtime_protos:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+      - name: Clone networking-recipe
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          path: recipe
+
+      - name: Install prerequisites
+        run: |
+          sudo apt install $PREREQS
+
+      - name: Install python dependencies
+        working-directory: recipe
+        run: |
+          python -m pip install setuptools build wheel
+
+      - name: Set up Go environment
+        uses: actions/setup-go@v4.1.0
+        with:
+          go-version: 'stable'
+
+      - name: Install DPDK SDE
+        uses: robinraju/release-downloader@v1.8
+        with:
+          repository: ${{ env.SDE_REPOSITORY }}
+          tag: ${{ env.SDE_TAG }}
+          fileName: ${{ env.SDE_FILENAME }}
+      - run: |
+          sudo tar -xzf $SDE_FILENAME -C /
+          rm $SDE_FILENAME
+
+      - name: Install gRPC plugins
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2
+
+      - name: Install stratum dependencies
+        uses: robinraju/release-downloader@v1.8
+        with:
+          repository: ${{ env.DEPS_REPOSITORY }}
+          tag: ${{ env.DEPS_TAG }}
+          fileName: ${{ env.DEPS_FILENAME }}
+      - run: |
+          sudo tar -xzf $DEPS_FILENAME -C /
+          rm $DEPS_FILENAME
+
+      - name: Build p4runtime protobufs
+        working-directory: recipe
+        run: |
+          export DEPEND_INSTALL=$DEPS_INSTALL_DIR
+          export SDE_INSTALL=$SDE_INSTALL_DIR
+          ./make-all.sh --target=dpdk --no-ovs --no-build
+          cmake --build build --target protobufs

.gitignore

@@ -1,4 +1,4 @@
 .vscode
 .vslick
-/build
-/install
+build
+install

CMakeLists.txt

@@ -14,6 +14,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 include(CMakePrintHelpers)
 include(CompilerSettings)
 include(CTest)
+include(ExternalProject)
 include(FindPkgConfig)
 include(GNUInstallDirs)
 
@@ -146,8 +147,17 @@ cmake_print_variables(CMAKE_PREFIX_PATH)
 # External packages #
 #####################
 
+# Stratum dependencies for the target system.
 include(StratumDependencies)
 
+# Protobuf compiler for the development system.
+find_package(HostProtoc)
+
+# TLS libraries.
+find_package(OpenSSL REQUIRED)
+mark_as_advanced(OpenSSL_DIR)
+
+# P4 SDE for the target system.
 if(DPDK_TARGET)
     find_package(DpdkDriver)
 elseif(ES2K_TARGET)
@@ -156,6 +166,7 @@ elseif(TOFINO_TARGET)
     find_package(TofinoDriver)
 endif()
 
+# Open vSwitch.
 if(WITH_OVSP4RT)
     find_package(OVS)
 endif()
@@ -243,6 +254,12 @@ function(set_install_rpath TARGET)
     endif()
 endfunction()
 
+#####################
+# Protobufs project #
+#####################
+
+include(P4RuntimeProtobufs)
+
 ##################
 # Subdirectories #
 ##################

cmake/FindHostProtoc.cmake

@@ -0,0 +1,90 @@
+# Import host Protobuf compiler.
+#
+# Copyright 2022-2023 Intel Corporation
+# SPDX-License-Identifier: Apache 2.0
+#
+
+include_guard(GLOBAL)
+
+#-----------------------------------------------------------------------
+# Protobuf compiler.
+# Runs on the development system.
+#-----------------------------------------------------------------------
+find_program(HOST_PROTOC_COMMAND "protoc" NO_CMAKE_FIND_ROOT_PATH)
+mark_as_advanced(HOST_PROTOC_COMMAND)
+
+if(HOST_PROTOC_COMMAND)
+  execute_process(
+    COMMAND ${HOST_PROTOC_COMMAND} --version
+    OUTPUT_VARIABLE protoc_output
+  )
+  string(STRIP "${protoc_output}" protoc_output)
+  string(REGEX REPLACE
+    "^libprotoc +([0-9.]+)$" "\\1" PROTOC_VERSION ${protoc_output})
+  message(STATUS "Found protoc: ${HOST_PROTOC_COMMAND} (version found \"${PROTOC_VERSION}\")")
+  unset(protoc_output)
+else()
+  message(FATAL_ERROR "protoc not found")
+endif()
+
+#-----------------------------------------------------------------------
+# gRPC c++ plugin for the Protobuf compiler.
+# Runs on the development (host) system.
+#-----------------------------------------------------------------------
+find_program(HOST_GRPC_CPP_PLUGIN "grpc_cpp_plugin" NO_CMAKE_FIND_ROOT_PATH)
+if(HOST_GRPC_CPP_PLUGIN)
+  message(STATUS "Found grpc_cpp_plugin")
+else()
+  message(FATAL_ERROR "grpc_cpp_plugin not found")
+endif()
+mark_as_advanced(HOST_GRPC_CPP_PLUGIN)
+
+#-----------------------------------------------------------------------
+# gRPC Python plugin for the Protobuf compiler.
+# Runs on the development (host) system.
+#-----------------------------------------------------------------------
+find_program(HOST_GRPC_PY_PLUGIN "grpc_python_plugin" NO_CMAKE_FIND_ROOT_PATH)
+if(HOST_GRPC_PY_PLUGIN)
+  message(STATUS "Found grpc_python_plugin")
+else()
+  message(WARNING "grpc_python_plugin not found")
+endif()
+mark_as_advanced(HOST_GRPC_PY_PLUGIN)
+
+#-----------------------------------------------------------------------
+# Go plugin for the Protobuf compiler.
+# Runs on the development (host) system.
+#-----------------------------------------------------------------------
+find_program(HOST_PROTOC_GO_PLUGIN "protoc-gen-go" NO_CMAKE_FIND_ROOT_PATH)
+mark_as_advanced(HOST_PROTOC_GO_PLUGIN)
+
+if(HOST_PROTOC_GO_PLUGIN)
+  execute_process(
+    COMMAND ${HOST_PROTOC_GO_PLUGIN} --version
+    OUTPUT_VARIABLE plugin_output
+  )
+  string(STRIP "${plugin_output}" plugin_output)
+  message(STATUS "Found ${plugin_output}")
+  unset(plugin_output)
+else()
+  message(STATUS "protoc-gen-go not found")
+endif()
+
+#-----------------------------------------------------------------------
+# Go gRPC plugin for the Protobuf compiler.
+# Runs on the development (host) system.
+#-----------------------------------------------------------------------
+find_program(HOST_GRPC_GO_PLUGIN "protoc-gen-go-grpc" NO_CMAKE_FIND_ROOT_PATH)
+mark_as_advanced(HOST_GRPC_GO_PLUGIN)
+
+if(HOST_GRPC_GO_PLUGIN)
+  execute_process(
+    COMMAND ${HOST_GRPC_GO_PLUGIN} --version
+    OUTPUT_VARIABLE plugin_output
+  )
+  string(STRIP "${plugin_output}" plugin_output)
+  message(STATUS "Found ${plugin_output}")
+  unset(plugin_output)
+else()
+  message(STATUS "protoc-gen-go-grpc plugin not found")
+endif()

cmake/P4RuntimeProtobufs.cmake

@@ -0,0 +1,28 @@
+#
+# Copyright 2022-2023 Intel Corporation
+# SPDX-License-Identifier: Apache 2.0
+#
+# Compile p4runtime protobufs for use by clients.
+#
+
+if(DEFINED GEN_GO_PROTOBUFS)
+  # Don't forward option to external project if it's undefined.
+  set(gen_go_protobufs -DGEN_GO_PROTOBUFS=${GEN_GO_PROTOBUFS})
+endif()
+
+ExternalProject_Add(protobufs
+  SOURCE_DIR
+    ${CMAKE_CURRENT_SOURCE_DIR}/protobufs
+  CMAKE_ARGS
+    -DDEPEND_INSTALL_DIR=${DEPEND_INSTALL_DIR}
+    -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
+    -DCPP_OUT=${CMAKE_CURRENT_BINARY_DIR}/cpp_out
+    -DGO_OUT=${CMAKE_CURRENT_BINARY_DIR}/go_out
+    -DPY_OUT=${CMAKE_CURRENT_BINARY_DIR}/py_out
+    ${gen_go_protobufs}
+  INSTALL_COMMAND
+    ${CMAKE_MAKE_PROGRAM} install
+  EXCLUDE_FROM_ALL TRUE
+)
+
+unset(gen_go_protobufs)

cmake/ProtobufCompile.cmake

@@ -13,7 +13,9 @@
 #   Directory in which the input .proto files should be installed
 # HOST_GRPC_CPP_PLUGIN
 #   Path to the host grpc_cpp_plugin executable
-# HOST_PROTOC
+# HOST_GRPC_PY_PLUGIN
+#   Path to the host grpc_python_plugin executable
+# HOST_PROTOC_COMMAND
 #   Path to the host protobuf compiler
 # PB_HEADER_INSTALL_DIR
 #   Directory in which the generated pb.h files should be installed
@@ -45,7 +47,7 @@ function(generate_proto_files PROTO_FILES SRC_DIR)
       OUTPUT
         ${_src} ${_hdr}
       COMMAND
-        ${HOST_PROTOC}
+        ${HOST_PROTOC_COMMAND}
         --proto_path=${PROTO_IMPORT_PATH}
         --cpp_out=${PB_OUT_DIR}
         -I${STRATUM_SOURCE_DIR}
@@ -97,7 +99,7 @@ function(generate_grpc_files PROTO_FILES SRC_DIR)
       OUTPUT
         ${_src} ${_hdr}
       COMMAND
-        ${HOST_PROTOC}
+        ${HOST_PROTOC_COMMAND}
         --proto_path=${PROTO_IMPORT_PATH}
         --grpc_out=${PB_OUT_DIR}
         --plugin=protoc-gen-grpc=${HOST_GRPC_CPP_PLUGIN}

cmake/StratumDependencies.cmake

@@ -40,6 +40,8 @@ mark_as_advanced(glog_DIR)
 find_package(Protobuf CONFIG REQUIRED)
 mark_as_advanced(Protobuf_DIR)
 
+message(STATUS "Found Protobuf version ${Protobuf_VERSION}")
+
 #-----------------------------------------------------------------------
 # Google RPC (gRPC).
 #-----------------------------------------------------------------------
@@ -48,37 +50,3 @@ mark_as_advanced(gRPC_DIR)
 mark_as_advanced(c-ares_DIR)
 
 message(STATUS "Found gRPC version ${gRPC_VERSION}")
-
-#-----------------------------------------------------------------------
-# Protobuf compiler.
-# Runs on the development system.
-#-----------------------------------------------------------------------
-find_program(HOST_PROTOC "protoc" NO_CMAKE_FIND_ROOT_PATH)
-mark_as_advanced(HOST_PROTOC)
-
-if(HOST_PROTOC)
-  message(STATUS "Found protoc: ${HOST_PROTOC}")
-else()
-  message(FATAL_ERROR "protoc not found")
-endif()
-
-#-----------------------------------------------------------------------
-# gRPC plugin for Protobuf compiler.
-# Runs on the development system.
-#-----------------------------------------------------------------------
-find_program(HOST_GRPC_CPP_PLUGIN "grpc_cpp_plugin" NO_CMAKE_FIND_ROOT_PATH)
-mark_as_advanced(HOST_GRPC_CPP_PLUGIN)
-
-if(HOST_GRPC_CPP_PLUGIN)
-  message(STATUS "Found grpc_cpp_plugin: ${HOST_GRPC_CPP_PLUGIN}")
-else()
-  message(FATAL_ERROR "grpc_cpp_plugin not found")
-endif()
-
-#-----------------------------------------------------------------------
-# SSL/TLS library (OpenSSL).
-#-----------------------------------------------------------------------
-find_package(OpenSSL REQUIRED)
-mark_as_advanced(OpenSSL_DIR)
-
-set(WITH_OPENSSL TRUE)

protobufs/.gitignore

@@ -0,0 +1 @@
+/build