diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 09e6a3eca5..f5c6b761ad 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -185,7 +185,7 @@ jobs: uses: actions/checkout@v2 - name: Checkout code and update modm tools run: | - (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm & wait + (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm protobuf==3.20.1 grpcio-tools & wait - name: Examples STM32F4 Without Discovery Board if: always() run: | diff --git a/.gitmodules b/.gitmodules index fae4f867ba..0be56feeac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -40,3 +40,6 @@ [submodule "ext/eyalroz/printf"] path = ext/eyalroz/printf url = https://github.com/modm-ext/printf-partial.git +[submodule "ext/nanopb/nanopb"] + path = ext/nanopb/nanopb + url = https://github.com/modm-ext/nanopb-partial.git diff --git a/examples/nucleo_f429zi/nanopb/main.cpp b/examples/nucleo_f429zi/nanopb/main.cpp new file mode 100644 index 0000000000..995f2da021 --- /dev/null +++ b/examples/nucleo_f429zi/nanopb/main.cpp @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2022, Lucas Moesch + * Copyright (c) 2022, Niklas Hauser + * + * This file is part of the modm project. + * + * This file originated from the nanopb project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include +#include +#include +#include +#include + +int main() +{ + Board::initialize(); + + /* This is the buffer where we will store our message. */ + uint8_t buffer[128]; + size_t message_length; + bool status; + + { + /* Allocate space on the stack to store the message data. + * + * Nanopb generates simple struct definitions for all the messages. + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + SimpleMessage message = SimpleMessage_init_zero; + + /* Create a stream that will write to our buffer. */ + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Fill in the lucky number */ + message.lucky_number = 42; + + /* Now we are ready to encode the message! */ + status = pb_encode(&stream, SimpleMessage_fields, &message); + modm_assert(status, "pb.enc", "Encoding SimpleMessage failed!"); + message_length = stream.bytes_written; + modm_assert(message_length, "pb.len", "Empty SimpleMessage buffer!"); + } + { + /* Now we could transmit the message over network, store it in a file or + * wrap it to a pigeon's leg. + */ + + /* But because we are lazy, we will just decode it immediately. */ + + /* Allocate space for the decoded message. */ + SimpleMessage message = SimpleMessage_init_zero; + + /* Create a stream that reads from the buffer. */ + pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); + + /* Now we are ready to decode the message. */ + status = pb_decode(&stream, SimpleMessage_fields, &message); + modm_assert(status, "pb.dec", "Decoding SimpleMessage failed!"); + modm_assert(message.lucky_number == 42, "lucky_number", "Incorrect SimpleMessage values!"); + } + + { + /* Allocate space on the stack to store the message data. + * + * Nanopb generates simple struct definitions for all the messages. + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + ComplexMessage message = ComplexMessage_init_zero; + + /* Create a stream that will write to our buffer. */ + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Fill in the unlucky number */ + std::strcpy(message.query, "Hello World"); + message.unlucky_number = 13; + message.toggle = true; + message.value = 4.00012f; + + /* Now we are ready to encode the message! */ + status = pb_encode(&stream, ComplexMessage_fields, &message); + modm_assert(status, "pb.enc", "Encoding ComplexMessage failed!"); + message_length = stream.bytes_written; + modm_assert(message_length, "pb.len", "Empty ComplexMessage buffer!"); + } + { + /* Now we could transmit the message over network, store it in a file or + * wrap it to a pigeon's leg. + */ + + /* But because we are lazy, we will just decode it immediately. */ + + /* Allocate space for the decoded message. */ + ComplexMessage message = ComplexMessage_init_zero; + + /* Create a stream that reads from the buffer. */ + pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); + + /* Now we are ready to decode the message. */ + status = pb_decode(&stream, ComplexMessage_fields, &message); + modm_assert(status, "pb.dec", "Decoding ComplexMessage failed!"); + + modm_assert(!strcmp(message.query, "Hello World"), "query", "Incorrect ComplexMessage values!"); + modm_assert(message.unlucky_number == 13, "unlucky_number", "Incorrect ComplexMessage values!"); + modm_assert(message.toggle == true, "toggle", "Incorrect ComplexMessage values!"); + modm_assert(int(message.value) == 4, "value", "Incorrect ComplexMessage values!"); + } + + while (true) + { + /* tumbleweed */ + } + return 0; +} diff --git a/examples/nucleo_f429zi/nanopb/project.xml b/examples/nucleo_f429zi/nanopb/project.xml new file mode 100644 index 0000000000..aef80dfb2f --- /dev/null +++ b/examples/nucleo_f429zi/nanopb/project.xml @@ -0,0 +1,11 @@ + + modm:nucleo-f429zi + + + + + + modm:build:scons + modm:nanopb + + diff --git a/examples/nucleo_f429zi/nanopb/protocol/complex.proto b/examples/nucleo_f429zi/nanopb/protocol/complex.proto new file mode 100644 index 0000000000..3aac1552a0 --- /dev/null +++ b/examples/nucleo_f429zi/nanopb/protocol/complex.proto @@ -0,0 +1,21 @@ +// -*- coding: utf-8 -*- +// +// Copyright (c) 2022, Niklas Hauser +// +// This file is part of the modm project. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// ----------------------------------------------------------------------------- +syntax = "proto3"; + +import "nanopb.proto"; + +message ComplexMessage { + string query = 1 [(nanopb).max_size = 40]; + int32 unlucky_number = 2; + bool toggle = 3; + double value = 4; +} + diff --git a/examples/nucleo_f429zi/nanopb/protocol/simple.proto b/examples/nucleo_f429zi/nanopb/protocol/simple.proto new file mode 100644 index 0000000000..20d031044e --- /dev/null +++ b/examples/nucleo_f429zi/nanopb/protocol/simple.proto @@ -0,0 +1,16 @@ +// -*- coding: utf-8 -*- +// +// Copyright (c) 2022, Lucas Moesch +// +// This file is part of the modm project. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// ----------------------------------------------------------------------------- +syntax = "proto3"; + +message SimpleMessage { + int32 lucky_number = 1; +} + diff --git a/ext/nanopb/nanopb b/ext/nanopb/nanopb new file mode 160000 index 0000000000..d5f15c1924 --- /dev/null +++ b/ext/nanopb/nanopb @@ -0,0 +1 @@ +Subproject commit d5f15c1924ed5fe0c19c9ea7be46bcde62bc4617 diff --git a/ext/nanopb/nanopb.lb b/ext/nanopb/nanopb.lb new file mode 100644 index 0000000000..4bea31671c --- /dev/null +++ b/ext/nanopb/nanopb.lb @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2021, Lucas Moesch +# Copyright (c) 2022, Niklas Hauser +# +# This file is part of the modm project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# ----------------------------------------------------------------------------- +import os +import subprocess +import tempfile + +from pathlib import Path + +def init(module): + module.name = ":nanopb" + module.description = """ +# Nanopb - Protocol Buffers for Embedded Systems + +Nanopb is a small code-size Protocol Buffers (protobuf) implementation in ansi +C. It is especially suitable for use in microcontrollers, but fits any memory +restricted system. + +See https://github.com/nanopb/nanopb. + + +## Build System Integration + +You can optionally point the build system to multiple protofiles using a +comma-separated list of paths in the `modm:nanopb:source` option. +You can specify the output location using the `modm:nanopb:path` option. +See the `modm:build` submodules for further details. + +!!! bug "Currently only with SCons support" + Only the `modm:build:scons` module currently supports this feature. +""" + +def prepare(module, options): + module.add_set_option( + PathOption(name="sources", absolute=True, empty_ok=True, + description="Comma-separated list of paths to protofiles."), + default="") + module.add_option( + PathOption(name="path", default="generated/nanopb", absolute=True, + description="Path to the generated messages folder")) + return True + +def build(env): + env.collect(":build:path.include", "modm/ext/nanopb") + env.outbasepath = "modm/ext/nanopb" + + env.copy("nanopb/pb_common.c", dest="pb_common.c") + env.copy("nanopb/pb_common.h", dest="pb_common.h") + env.copy("nanopb/pb_decode.c", dest="pb_decode.c") + env.copy("nanopb/pb_decode.h", dest="pb_decode.h") + env.copy("nanopb/pb_encode.c", dest="pb_encode.c") + env.copy("nanopb/pb_encode.h", dest="pb_encode.h") + env.copy("nanopb/pb.h", dest="pb.h") diff --git a/tools/build_script_generator/scons/module.lb b/tools/build_script_generator/scons/module.lb index 2e5f8c224e..e605e611e0 100644 --- a/tools/build_script_generator/scons/module.lb +++ b/tools/build_script_generator/scons/module.lb @@ -3,6 +3,7 @@ # # Copyright (c) 2017-2018, Niklas Hauser # Copyright (c) 2019, Raphael Lehmann +# Copyright (c) 2022, Lucas Moesch # # This file is part of the modm project. # @@ -108,6 +109,9 @@ def build(env): elif device["core"].startswith("avr"): tools.update({"size", "avrdude"}) + if env.has_module(":nanopb"): + tools.add("nanopb") + env.collect("path.tools", *toolpaths) env.collect("tools", *tools) @@ -117,9 +121,15 @@ def build(env): path = "site_tools/{}.py".format(tool) if exists(localpath(path)): env.copy(path) + # Copy support files env.copy("site_tools/qtcreator/") + if env.has_module(":nanopb"): + env.copy("site_tools/nanopb_builder/") + env.copy(repopath("ext/nanopb/nanopb/generator"), dest="site_tools/nanopb_builder/generator") + env.copy(repopath("ext/nanopb/nanopb/tests/site_scons/site_tools/nanopb.py"), dest="site_tools/nanopb_builder/nanopb.py") + # Generate the env.BuildTarget tool linkerscript = env.get(":platform:cortex-m:linkerscript.override") linkerscript = env.relcwdoutpath(linkerscript) if linkerscript \ @@ -139,6 +149,7 @@ def post_build(env): is_unittest = len(env["::unittest.source"]) has_xpcc_generator = env.has_module(":communication:xpcc:generator") has_image_source = len(env["::image.source"]) + has_nanopb = env.has_module(":nanopb") repositories = [p for p in env.buildlog.repositories if isdir(env.real_outpath(p, basepath="."))] repositories.sort(key=lambda name: "0" if name == "modm" else name) generated_paths = [join(env.relcwdoutpath(""), r) for r in repositories] @@ -161,9 +172,9 @@ def post_build(env): "artifact_path": env.relcwdoutpath(env["path.artifact"]), "generated_paths": generated_paths, "is_unittest": is_unittest, - "has_image_source": has_image_source, "has_xpcc_generator": has_xpcc_generator, + "has_nanopb": has_nanopb, }) if has_image_source: subs["image_source"] = env.relcwdoutpath(env["::image.source"]) @@ -176,6 +187,9 @@ def post_build(env): "generator_path": env.relcwdoutpath(env.get(":communication:xpcc:generator:path", "")), "generator_namespace": env.get(":communication:xpcc:generator:namespace", ""), }) + if has_nanopb: + subs["nanopb_source"] = [env.relcwdoutpath(p) for p in env.get(":nanopb:sources", [])] + subs["nanopb_path"] = env.relcwdoutpath(env.get(":nanopb:path", ".")) if subs["platform"] == "avr": subs.update(env.query("::avrdude_options")) if subs["platform"] == "sam": @@ -220,6 +234,7 @@ def post_build(env): "tools": tools, "is_modm": repo == "modm", }) + # Generate library SConscript env.outbasepath = repo env.template("resources/SConscript.in", "SConscript", diff --git a/tools/build_script_generator/scons/module.md b/tools/build_script_generator/scons/module.md index 945efb932a..63550794a6 100644 --- a/tools/build_script_generator/scons/module.md +++ b/tools/build_script_generator/scons/module.md @@ -600,6 +600,24 @@ are supported, this is only meant for using the IDE as an editor. !!! warning "Consider this an unstable feature" +## Protobuf Generator Tool + +The `modm:nanopb` module contains a Python generator to translate the messages +defined in `*.proto` files by the `modm:nanopb:source` option into `*.pb.cpp` +and `*.pb.hpp` files. +This module contains a SCons wrapper tool that automatically updates the +generated files when it becomes necessary: + +```py +cpp_sources += env.NanopbProtofile( + sources=options[":nanopb:sources"], + path=options[":nanopb:path"], +) +``` + +The generated files are available as a top-level `#include `. + + ## XPCC Generator Tool The `modm:communication:xpcc:generator` module contains the Python tools to @@ -611,7 +629,7 @@ The wrapper tool is automatically used when the generator module is detected, and its options are evaluated for the wrapper as follows: ```py -env.XpccCommunication( +cpp_sources += env.XpccCommunication( xmlfile=options["::xpcc:generator:source"], container=options["::xpcc:generator:container"], path=options["::xpcc:generator:path"], diff --git a/tools/build_script_generator/scons/resources/SConstruct.in b/tools/build_script_generator/scons/resources/SConstruct.in index 9bea04abb0..9b5aa2b5f1 100644 --- a/tools/build_script_generator/scons/resources/SConstruct.in +++ b/tools/build_script_generator/scons/resources/SConstruct.in @@ -66,6 +66,14 @@ sources += env.XpccCommunication( namespace="{{ generator_namespace }}", include_paths=["."]) %% endif +%% if has_nanopb +# Generating Nanopb messages +env.Append(CPPPATH=abspath("{{ nanopb_path }}")) +ignored.append("{{ nanopb_path }}") +sources += env.NanopbProtofile( + sources={{ nanopb_source }}, + path=abspath("{{ nanopb_path }}")) +%% endif %% if options["::info.git"] != "Disabled" sources.append(env.InfoGit(with_status={{ "True" if "Status" in options["::info.git"] else "False" }})) diff --git a/tools/build_script_generator/scons/site_tools/comstr.py b/tools/build_script_generator/scons/site_tools/comstr.py index 883b4b5326..c7745b8a0d 100644 --- a/tools/build_script_generator/scons/site_tools/comstr.py +++ b/tools/build_script_generator/scons/site_tools/comstr.py @@ -96,6 +96,9 @@ def generate(env, **kw): env["XPCC_TASK_CALLER_COMSTR"] = "%s╭─────XPCC───── %s$SOURCE\n" \ "%s╰─Task─Caller─> %s$TARGET%s" % template + env["NANOPB_MESSAGES_COMSTR"] = "%s╭────Nanopb──── %s$SOURCE\n" \ + "%s╰───Messages──> %s$TARGET%s" % template + # modm tools installing env["ARTIFACT_COMSTR"] = "%s╭───Artifact─── %s$SOURCE\n" \ "%s╰────Cache────> %s$ARTIFACT_FILEPATH%s" % install diff --git a/tools/build_script_generator/scons/site_tools/nanopb.py b/tools/build_script_generator/scons/site_tools/nanopb.py new file mode 100644 index 0000000000..47e2ba8ebf --- /dev/null +++ b/tools/build_script_generator/scons/site_tools/nanopb.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022, Lucas Moesch +# Copyright (c) 2022, Niklas Hauser +# +# This file is part of the modm project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# ----------------------------------------------------------------------------- + +import SCons.Action +import SCons.Builder +import SCons.Util +from SCons.Script import Dir, File + +import os.path +import platform +import sys + +import nanopb_builder + +# ----------------------------------------------------------------------------- +# Support for nanopb protobuf compilation. +# This suits as a wrapper for nanopb's own SCons build integration, which is +# integrated via the nanopb_builder module. The SCons integration can be found +# in modm/ext/nanopb/nanopb/tests/site_scons/site_tools/nanopb.py. +# ----------------------------------------------------------------------------- + +def relpath(path, start): + os_relpath = os.path.relpath(path, start) + if os_relpath[0] == ".": + return os_relpath + else: + return os.path.join(".", os_relpath) + + +def _nanopb_proto_actions(source, target, env, for_signature): + esc = env["ESCAPE"] + outpath = env.get("path", ".") + actionlist = [] + for protofile in source: + # Make protoc build inside the SConscript directory + srcfile = esc(os.path.basename(str(protofile))) + targetfile = os.path.relpath(os.path.join(outpath, protofile.name.replace(".proto", ".pb.cpp")), ".") + srcdir = esc(relpath(os.path.dirname(str(protofile)), ".")) + include_dirs = [srcdir] + + for d in env["PROTOCPATH"]: + d = env.GetBuildPath(d) + if not os.path.isabs(d): d = os.path.relpath(d, ".") + include_dirs.append(esc(d)) + + # when generating .pb.cpp sources, instead of pb.h generate .pb.hpp headers + nanopb_flags = ["--source-extension=.cpp", "--header-extension=.hpp"] + if env["NANOPBFLAGS"]: nanopb_flags.append(env["NANOPBFLAGS"]) + nanopb_flags = ",".join(nanopb_flags) + + actionlist.append(SCons.Action.CommandAction( + "$PROTOC $PROTOCFLAGS -I{} --nanopb_out={} --nanopb_opt={} {}" \ + .format(" -I".join(include_dirs), outpath, nanopb_flags, srcfile), + cmdstr=env.subst("$NANOPB_MESSAGES_COMSTR", target=targetfile, source=protofile))) + return SCons.Action.ListAction(actionlist) + + +def _nanopb_proto_emitter(target, source, env): + outpath = env.get("path", ".") + targets = [] + for protofile in source: + targets.append(os.path.join(outpath, protofile.name.replace(".proto", ".pb.cpp"))) + targets.append(os.path.join(outpath, protofile.name.replace(".proto", ".pb.hpp"))) + options = os.path.splitext(str(protofile))[0] + ".options" + if os.path.exists(options): + env.Depends(target=protofile, dependency=options) + + return targets, source + + +def build_proto(env, sources, path): + if not sources: return [] + files = env.NanopbProtoCpp(sources, path=path) + sources = [file for file in files if file.name.endswith(".cpp")] + return sources + + +def generate(env, **kw): + env["NANOPB"] = os.path.abspath("./scons/site_tools/nanopb_builder") + env["PROTOC"] = nanopb_builder._detect_protoc(env) + env["PROTOCFLAGS"] = nanopb_builder._detect_protocflags(env) + + env.SetDefault(NANOPBFLAGS = "") + env.SetDefault(PROTOCPATH = [".", os.path.join(env["NANOPB"], "generator", "proto")]) + + env["BUILDERS"]["NanopbProtoCpp"] = \ + SCons.Builder.Builder( + generator = _nanopb_proto_actions, + suffix = ".pb.cpp", + src_suffix = ".proto", + emitter = _nanopb_proto_emitter) + + env.AddMethod(build_proto, "NanopbProtofile") + +def exists(env): + return True diff --git a/tools/build_script_generator/scons/site_tools/nanopb_builder/__init__.py b/tools/build_script_generator/scons/site_tools/nanopb_builder/__init__.py new file mode 100644 index 0000000000..45a554f9d0 --- /dev/null +++ b/tools/build_script_generator/scons/site_tools/nanopb_builder/__init__.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2022, Lucas Moesch +# +# This file is part of the modm project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# ----------------------------------------------------------------------------- + +from .nanopb import _nanopb_proto_actions, _detect_protocflags, _detect_protoc