From b303c341ef4a2b8c3b20618f5df5a9d936d77bf7 Mon Sep 17 00:00:00 2001 From: David Drysdale Date: Fri, 26 Jun 2020 11:37:17 +0100 Subject: [PATCH 1/2] Add pristine local copy of prost Taken from https://github.com/danburkert/prost as of commit 802562779979 ("prost-build: fix compilation error when using --no-default-features"), with the exception of the prost-build/third-party subdirectory (which contains an unneeded copy of the protobuf distribution). --- .../continuous-integration-workflow.yaml | 111 + third_party/prost/.gitignore | 2 + third_party/prost/Cargo.toml | 60 + third_party/prost/LICENSE | 201 ++ third_party/prost/README.md | 368 ++++ third_party/prost/benches/varint.rs | 94 + third_party/prost/clippy.toml | 1 + third_party/prost/conformance/Cargo.toml | 14 + .../prost/conformance/failing_tests.txt | 3 + third_party/prost/conformance/src/main.rs | 115 ++ .../prost/conformance/tests/conformance.rs | 33 + third_party/prost/fuzz/.gitignore | 2 + third_party/prost/fuzz/Cargo.toml | 22 + third_party/prost/fuzz/fuzzers/proto2.rs | 10 + third_party/prost/fuzz/fuzzers/proto3.rs | 9 + third_party/prost/prepare-release.sh | 47 + third_party/prost/prost-build/Cargo.toml | 27 + third_party/prost/prost-build/README.md | 16 + third_party/prost/prost-build/build.rs | 104 + third_party/prost/prost-build/src/ast.rs | 132 ++ .../prost/prost-build/src/code_generator.rs | 1004 +++++++++ .../prost/prost-build/src/extern_paths.rs | 170 ++ .../prost/prost-build/src/goodbye.proto | 9 + third_party/prost/prost-build/src/hello.proto | 9 + third_party/prost/prost-build/src/ident.rs | 236 +++ third_party/prost/prost-build/src/lib.rs | 813 ++++++++ .../prost/prost-build/src/message_graph.rs | 91 + .../prost/prost-build/src/smoke_test.proto | 18 + third_party/prost/prost-build/src/types.proto | 11 + third_party/prost/prost-derive/Cargo.toml | 20 + third_party/prost/prost-derive/README.md | 16 + .../prost/prost-derive/src/field/group.rs | 134 ++ .../prost/prost-derive/src/field/map.rs | 394 ++++ .../prost/prost-derive/src/field/message.rs | 134 ++ .../prost/prost-derive/src/field/mod.rs | 366 ++++ .../prost/prost-derive/src/field/oneof.rs | 99 + .../prost/prost-derive/src/field/scalar.rs | 787 +++++++ third_party/prost/prost-derive/src/lib.rs | 475 +++++ third_party/prost/prost-types/Cargo.toml | 24 + third_party/prost/prost-types/README.md | 21 + third_party/prost/prost-types/src/compiler.rs | 123 ++ third_party/prost/prost-types/src/lib.rs | 177 ++ third_party/prost/prost-types/src/protobuf.rs | 1809 +++++++++++++++++ third_party/prost/protobuf/Cargo.toml | 31 + third_party/prost/protobuf/README.md | 8 + third_party/prost/protobuf/benches/dataset.rs | 113 + third_party/prost/protobuf/build.rs | 277 +++ ...-conformance_test_runner-cmake-build.patch | 41 + third_party/prost/protobuf/src/lib.rs | 116 ++ third_party/prost/publish-release.sh | 26 + third_party/prost/src/encoding.rs | 1665 +++++++++++++++ third_party/prost/src/error.rs | 116 ++ third_party/prost/src/lib.rs | 93 + third_party/prost/src/message.rs | 167 ++ third_party/prost/src/types.rs | 386 ++++ third_party/prost/tests-2015/Cargo.toml | 36 + third_party/prost/tests-no-std/Cargo.toml | 37 + third_party/prost/tests/Cargo.toml | 34 + third_party/prost/tests/src/bootstrap.rs | 95 + third_party/prost/tests/src/build.rs | 124 ++ .../prost/tests/src/custom_attributes.proto | 16 + third_party/prost/tests/src/debug.rs | 96 + .../prost/tests/src/default_enum_value.proto | 26 + .../prost/tests/src/deprecated_field.proto | 8 + .../prost/tests/src/deprecated_field.rs | 29 + third_party/prost/tests/src/extern_paths.rs | 45 + third_party/prost/tests/src/groups.proto | 57 + .../prost/tests/src/ident_conversion.proto | 81 + third_party/prost/tests/src/lib.rs | 582 ++++++ .../prost/tests/src/message_encoding.rs | 370 ++++ third_party/prost/tests/src/nesting.proto | 25 + third_party/prost/tests/src/no_package.proto | 1 + .../prost/tests/src/no_unused_results.proto | 7 + .../prost/tests/src/no_unused_results.rs | 7 + .../prost/tests/src/oneof_attributes.proto | 14 + .../prost/tests/src/packages/gizmo.proto | 8 + third_party/prost/tests/src/packages/mod.rs | 52 + .../prost/tests/src/packages/root.proto | 8 + .../prost/tests/src/packages/widget.proto | 14 + .../tests/src/packages/widget_factory.proto | 24 + .../prost/tests/src/recursive_oneof.proto | 17 + third_party/prost/tests/src/unittest.rs | 48 + .../prost/tests/src/well_known_types.proto | 13 + .../prost/tests/src/well_known_types.rs | 14 + 84 files changed, 13238 insertions(+) create mode 100644 third_party/prost/.github/workflows/continuous-integration-workflow.yaml create mode 100644 third_party/prost/.gitignore create mode 100644 third_party/prost/Cargo.toml create mode 100644 third_party/prost/LICENSE create mode 100644 third_party/prost/README.md create mode 100644 third_party/prost/benches/varint.rs create mode 100644 third_party/prost/clippy.toml create mode 100644 third_party/prost/conformance/Cargo.toml create mode 100644 third_party/prost/conformance/failing_tests.txt create mode 100644 third_party/prost/conformance/src/main.rs create mode 100644 third_party/prost/conformance/tests/conformance.rs create mode 100644 third_party/prost/fuzz/.gitignore create mode 100644 third_party/prost/fuzz/Cargo.toml create mode 100644 third_party/prost/fuzz/fuzzers/proto2.rs create mode 100644 third_party/prost/fuzz/fuzzers/proto3.rs create mode 100755 third_party/prost/prepare-release.sh create mode 100644 third_party/prost/prost-build/Cargo.toml create mode 100644 third_party/prost/prost-build/README.md create mode 100644 third_party/prost/prost-build/build.rs create mode 100644 third_party/prost/prost-build/src/ast.rs create mode 100644 third_party/prost/prost-build/src/code_generator.rs create mode 100644 third_party/prost/prost-build/src/extern_paths.rs create mode 100644 third_party/prost/prost-build/src/goodbye.proto create mode 100644 third_party/prost/prost-build/src/hello.proto create mode 100644 third_party/prost/prost-build/src/ident.rs create mode 100644 third_party/prost/prost-build/src/lib.rs create mode 100644 third_party/prost/prost-build/src/message_graph.rs create mode 100644 third_party/prost/prost-build/src/smoke_test.proto create mode 100644 third_party/prost/prost-build/src/types.proto create mode 100644 third_party/prost/prost-derive/Cargo.toml create mode 100644 third_party/prost/prost-derive/README.md create mode 100644 third_party/prost/prost-derive/src/field/group.rs create mode 100644 third_party/prost/prost-derive/src/field/map.rs create mode 100644 third_party/prost/prost-derive/src/field/message.rs create mode 100644 third_party/prost/prost-derive/src/field/mod.rs create mode 100644 third_party/prost/prost-derive/src/field/oneof.rs create mode 100644 third_party/prost/prost-derive/src/field/scalar.rs create mode 100644 third_party/prost/prost-derive/src/lib.rs create mode 100644 third_party/prost/prost-types/Cargo.toml create mode 100644 third_party/prost/prost-types/README.md create mode 100644 third_party/prost/prost-types/src/compiler.rs create mode 100644 third_party/prost/prost-types/src/lib.rs create mode 100644 third_party/prost/prost-types/src/protobuf.rs create mode 100644 third_party/prost/protobuf/Cargo.toml create mode 100644 third_party/prost/protobuf/README.md create mode 100644 third_party/prost/protobuf/benches/dataset.rs create mode 100644 third_party/prost/protobuf/build.rs create mode 100644 third_party/prost/protobuf/src/fix-conformance_test_runner-cmake-build.patch create mode 100644 third_party/prost/protobuf/src/lib.rs create mode 100755 third_party/prost/publish-release.sh create mode 100644 third_party/prost/src/encoding.rs create mode 100644 third_party/prost/src/error.rs create mode 100644 third_party/prost/src/lib.rs create mode 100644 third_party/prost/src/message.rs create mode 100644 third_party/prost/src/types.rs create mode 100644 third_party/prost/tests-2015/Cargo.toml create mode 100644 third_party/prost/tests-no-std/Cargo.toml create mode 100644 third_party/prost/tests/Cargo.toml create mode 100644 third_party/prost/tests/src/bootstrap.rs create mode 100644 third_party/prost/tests/src/build.rs create mode 100644 third_party/prost/tests/src/custom_attributes.proto create mode 100644 third_party/prost/tests/src/debug.rs create mode 100644 third_party/prost/tests/src/default_enum_value.proto create mode 100644 third_party/prost/tests/src/deprecated_field.proto create mode 100644 third_party/prost/tests/src/deprecated_field.rs create mode 100644 third_party/prost/tests/src/extern_paths.rs create mode 100644 third_party/prost/tests/src/groups.proto create mode 100644 third_party/prost/tests/src/ident_conversion.proto create mode 100644 third_party/prost/tests/src/lib.rs create mode 100644 third_party/prost/tests/src/message_encoding.rs create mode 100644 third_party/prost/tests/src/nesting.proto create mode 100644 third_party/prost/tests/src/no_package.proto create mode 100644 third_party/prost/tests/src/no_unused_results.proto create mode 100644 third_party/prost/tests/src/no_unused_results.rs create mode 100644 third_party/prost/tests/src/oneof_attributes.proto create mode 100644 third_party/prost/tests/src/packages/gizmo.proto create mode 100644 third_party/prost/tests/src/packages/mod.rs create mode 100644 third_party/prost/tests/src/packages/root.proto create mode 100644 third_party/prost/tests/src/packages/widget.proto create mode 100644 third_party/prost/tests/src/packages/widget_factory.proto create mode 100644 third_party/prost/tests/src/recursive_oneof.proto create mode 100644 third_party/prost/tests/src/unittest.rs create mode 100644 third_party/prost/tests/src/well_known_types.proto create mode 100644 third_party/prost/tests/src/well_known_types.rs diff --git a/third_party/prost/.github/workflows/continuous-integration-workflow.yaml b/third_party/prost/.github/workflows/continuous-integration-workflow.yaml new file mode 100644 index 00000000000..3182d2c3d8d --- /dev/null +++ b/third_party/prost/.github/workflows/continuous-integration-workflow.yaml @@ -0,0 +1,111 @@ +name: continuous integration +on: pull_request + +jobs: + + rustfmt: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + default: true + profile: minimal + components: rustfmt + - name: rustfmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + default: true + profile: minimal + components: clippy + - name: install ninja + uses: seanmiddleditch/gha-setup-ninja@v1 + - name: clippy + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --workspace --all-features --all-targets + + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + toolchain: + - stable + - 1.39.0 + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + default: true + profile: minimal + - name: install ninja + uses: seanmiddleditch/gha-setup-ninja@v1 + - name: test + uses: actions-rs/cargo@v1 + with: + command: test + args: --workspace --all-targets + - name: test no-default-features + uses: actions-rs/cargo@v1 + with: + command: test + args: --no-default-features + + no-std: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v2 + - name: install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + default: true + profile: minimal + - name: install cargo-no-std-check + uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-no-std-check + - name: prost cargo-no-std-check + uses: actions-rs/cargo@v1 + with: + command: no-std-check + args: --manifest-path Cargo.toml --no-default-features + - name: prost-types cargo-no-std-check + uses: actions-rs/cargo@v1 + with: + command: no-std-check + args: --manifest-path prost-types/Cargo.toml --no-default-features + # prost-build depends on prost with --no-default-features, but when + # prost-build is built through the workspace, prost typically has default + # features enabled due to vagaries in Cargo workspace feature resolution. + # This additional check ensures that prost-build does not rely on any of + # prost's default features to compile. + - name: prost-build check + uses: actions-rs/cargo@v1 + with: + command: check + args: --manifest-path prost-build/Cargo.toml diff --git a/third_party/prost/.gitignore b/third_party/prost/.gitignore new file mode 100644 index 00000000000..a9d37c560c6 --- /dev/null +++ b/third_party/prost/.gitignore @@ -0,0 +1,2 @@ +target +Cargo.lock diff --git a/third_party/prost/Cargo.toml b/third_party/prost/Cargo.toml new file mode 100644 index 00000000000..1701b4489dc --- /dev/null +++ b/third_party/prost/Cargo.toml @@ -0,0 +1,60 @@ +[package] +name = "prost" +version = "0.6.1" +authors = ["Dan Burkert "] +license = "Apache-2.0" +repository = "https://github.com/danburkert/prost" +documentation = "https://docs.rs/prost" +readme = "README.md" +description = "A Protocol Buffers implementation for the Rust Language." +keywords = ["protobuf", "serialization"] +categories = ["encoding"] +edition = "2018" + +[badges] +travis-ci = { repository = "danburkert/prost" } +appveyor = { repository = "danburkert/prost" } + +[workspace] +members = [ + "conformance", + "prost-build", + "prost-derive", + "prost-types", + "protobuf", + "tests", + "tests-2015", + "tests-no-std", +] +exclude = [ + # The fuzz crate can't be compiled or tested without the 'cargo fuzz' command, + # so exclude it from normal builds. + "fuzz", +] + +[lib] +# https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options +bench = false + +[features] +default = ["prost-derive", "std"] +no-recursion-limit = [] +std = [] + +[dependencies] +bytes = { version = "0.5", default-features = false } +prost-derive = { version = "0.6.1", path = "prost-derive", optional = true } + +[dev-dependencies] +criterion = "0.3" +env_logger = { version = "0.7", default-features = false } +log = "0.4" +quickcheck = "0.9" +rand = "0.7" + +[profile.bench] +debug = true + +[[bench]] +name = "varint" +harness = false diff --git a/third_party/prost/LICENSE b/third_party/prost/LICENSE new file mode 100644 index 00000000000..16fe87b06e8 --- /dev/null +++ b/third_party/prost/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/prost/README.md b/third_party/prost/README.md new file mode 100644 index 00000000000..20de748350b --- /dev/null +++ b/third_party/prost/README.md @@ -0,0 +1,368 @@ +[![Build Status](https://travis-ci.org/danburkert/prost.svg?branch=master)](https://travis-ci.org/danburkert/prost) +[![Windows Build Status](https://ci.appveyor.com/api/projects/status/24rpba3x2vqe8lje/branch/master?svg=true)](https://ci.appveyor.com/project/danburkert/prost/branch/master) +[![Documentation](https://docs.rs/prost/badge.svg)](https://docs.rs/prost/) +[![Crate](https://img.shields.io/crates/v/prost.svg)](https://crates.io/crates/prost) +[![Dependency Status](https://deps.rs/repo/github/danburkert/prost/status.svg)](https://deps.rs/repo/github/danburkert/prost) + +# *PROST!* + +`prost` is a [Protocol Buffers](https://developers.google.com/protocol-buffers/) +implementation for the [Rust Language](https://www.rust-lang.org/). `prost` +generates simple, idiomatic Rust code from `proto2` and `proto3` files. + +Compared to other Protocol Buffers implementations, `prost` + +* Generates simple, idiomatic, and readable Rust types by taking advantage of + Rust `derive` attributes. +* Retains comments from `.proto` files in generated Rust code. +* Allows existing Rust types (not generated from a `.proto`) to be serialized + and deserialized by adding attributes. +* Uses the [`bytes::{Buf, BufMut}`](https://github.com/carllerche/bytes) + abstractions for serialization instead of `std::io::{Read, Write}`. +* Respects the Protobuf `package` specifier when organizing generated code + into Rust modules. +* Preserves unknown enum values during deserialization. +* Does not include support for runtime reflection or message descriptors. + +## Using `prost` in a Cargo Project + +First, add `prost` and its public dependencies to your `Cargo.toml`: + +``` +[dependencies] +prost = "0.6" +# Only necessary if using Protobuf well-known types: +prost-types = "0.6" +``` + +The recommended way to add `.proto` compilation to a Cargo project is to use the +`prost-build` library. See the [`prost-build` documentation](prost-build) for +more details and examples. + +## Generated Code + +`prost` generates Rust code from source `.proto` files using the `proto2` or +`proto3` syntax. `prost`'s goal is to make the generated code as simple as +possible. + +### Packages + +All `.proto` files used with `prost` must contain a +[`package` specifier][package]. `prost` will translate the Protobuf package into +a Rust module. For example, given the `package` specifier: + +[package]: https://developers.google.com/protocol-buffers/docs/proto#packages + +```proto +package foo.bar; +``` + +All Rust types generated from the file will be in the `foo::bar` module. + +### Messages + +Given a simple message declaration: + +```proto +// Sample message. +message Foo { +} +``` + +`prost` will generate the following Rust struct: + +```rust +/// Sample message. +#[derive(Clone, Debug, PartialEq, Message)] +pub struct Foo { +} +``` + +### Fields + +Fields in Protobuf messages are translated into Rust as public struct fields of the +corresponding type. + +#### Scalar Values + +Scalar value types are converted as follows: + +| Protobuf Type | Rust Type | +| --- | --- | +| `double` | `f64` | +| `float` | `f32` | +| `int32` | `i32` | +| `int64` | `i64` | +| `uint32` | `u32` | +| `uint64` | `u64` | +| `sint32` | `i32` | +| `sint64` | `i64` | +| `fixed32` | `u32` | +| `fixed64` | `u64` | +| `sfixed32` | `i32` | +| `sfixed64` | `i64` | +| `bool` | `bool` | +| `string` | `String` | +| `bytes` | `Vec` | + +#### Enumerations + +All `.proto` enumeration types convert to the Rust `i32` type. Additionally, +each enumeration type gets a corresponding Rust `enum` type, with helper methods +to convert `i32` values to the enum type. The `enum` type isn't used directly as +a field, because the Protobuf spec mandates that enumerations values are 'open', +and decoding unrecognized enumeration values must be possible. + +#### Field Modifiers + +Protobuf scalar value and enumeration message fields can have a modifier +depending on the Protobuf version. Modifiers change the corresponding type of +the Rust field: + +| `.proto` Version | Modifier | Rust Type | +| --- | --- | --- | +| `proto2` | `optional` | `Option` | +| `proto2` | `required` | `T` | +| `proto3` | default | `T` | +| `proto2`/`proto3` | repeated | `Vec` | + +#### Map Fields + +Map fields are converted to a Rust `HashMap` with key and value type converted +from the Protobuf key and value types. + +#### Message Fields + +Message fields are converted to the corresponding struct type. The table of +field modifiers above applies to message fields, except that `proto3` message +fields without a modifier (the default) will be wrapped in an `Option`. +Typically message fields are unboxed. `prost` will automatically box a message +field if the field type and the parent type are recursively nested in order to +avoid an infinite sized struct. + +#### Oneof Fields + +Oneof fields convert to a Rust enum. Protobuf `oneof`s types are not named, so +`prost` uses the name of the `oneof` field for the resulting Rust enum, and +defines the enum in a module under the struct. For example, a `proto3` message +such as: + +```proto +message Foo { + oneof widget { + int32 quux = 1; + string bar = 2; + } +} +``` + +generates the following Rust[1]: + +```rust +pub struct Foo { + pub widget: Option, +} +pub mod foo { + pub enum Widget { + Quux(i32), + Bar(String), + } +} +``` + +`oneof` fields are always wrapped in an `Option`. + +[1] Annotations have been elided for clarity. See below for a full example. + +### Services + +`prost-build` allows a custom code-generator to be used for processing `service` +definitions. This can be used to output Rust traits according to an +application's specific needs. + +### Generated Code Example + +Example `.proto` file: + +```proto +syntax = "proto3"; +package tutorial; + +message Person { + string name = 1; + int32 id = 2; // Unique ID number for this person. + string email = 3; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + string number = 1; + PhoneType type = 2; + } + + repeated PhoneNumber phones = 4; +} + +// Our address book file is just one of these. +message AddressBook { + repeated Person people = 1; +} +``` + +and the generated Rust code (`tutorial.rs`): + +```rust +#[derive(Clone, Debug, PartialEq, Message)] +pub struct Person { + #[prost(string, tag="1")] + pub name: String, + /// Unique ID number for this person. + #[prost(int32, tag="2")] + pub id: i32, + #[prost(string, tag="3")] + pub email: String, + #[prost(message, repeated, tag="4")] + pub phones: Vec, +} +pub mod person { + #[derive(Clone, Debug, PartialEq, Message)] + pub struct PhoneNumber { + #[prost(string, tag="1")] + pub number: String, + #[prost(enumeration="PhoneType", tag="2")] + pub type_: i32, + } + #[derive(Clone, Copy, Debug, PartialEq, Eq, Enumeration)] + pub enum PhoneType { + Mobile = 0, + Home = 1, + Work = 2, + } +} +/// Our address book file is just one of these. +#[derive(Clone, Debug, PartialEq, Message)] +pub struct AddressBook { + #[prost(message, repeated, tag="1")] + pub people: Vec, +} +``` + +## Using `prost` in a `no_std` Crate + +`prost` is compatible with `no_std` crates. To enable `no_std` support, disable +the `std` features in `prost` and `prost-types`: + +``` +[dependencies] +prost = { version = "0.6", default-features = false, features = ["prost-derive"] } +# Only necessary if using Protobuf well-known types: +prost-types = { version = "0.6", default-features = false } +``` + +Additionally, configure `prost-buid` to output `BTreeMap`s instead of `HashMap`s +for all Protobuf `map` fields in your `build.rs`: + +```rust +let mut config = prost_build::Config::new(); +config.btree_map(&["."]); +``` + +When using edition 2015, it may be necessary to add an `extern crate core;` +directive to the crate which includes `prost`-generated code. + +## Serializing Existing Types + +`prost` uses a custom derive macro to handle encoding and decoding types, which +means that if your existing Rust type is compatible with Protobuf types, you can +serialize and deserialize it by adding the appropriate derive and field +annotations. + +Currently the best documentation on adding annotations is to look at the +generated code examples above. + +### Tag Inference for Existing Types + +Prost automatically infers tags for the struct. + +Fields are tagged sequentially in the order they +are specified, starting with `1`. + +You may skip tags which have been reserved, or where there are gaps between +sequentially occurring tag values by specifying the tag number to skip to with +the `tag` attribute on the first field after the gap. The following fields will +be tagged sequentially starting from the next number. + +```rust +#[derive(Clone, Debug, PartialEq, Message)] +struct Person { + pub id: String, // tag=1 + + // NOTE: Old "name" field has been removed + // pub name: String, // tag=2 (Removed) + + #[prost(tag="6")] + pub given_name: String, // tag=6 + pub family_name: String, // tag=7 + pub formatted_name: String, // tag=8 + + #[prost(tag="3")] + pub age: u32, // tag=3 + pub height: u32, // tag=4 + #[prost(enumeration="Gender")] + pub gender: i32, // tag=5 + + // NOTE: Skip to less commonly occurring fields + #[prost(tag="16")] + pub name_prefix: String, // tag=16 (eg. mr/mrs/ms) + pub name_suffix: String, // tag=17 (eg. jr/esq) + pub maiden_name: String, // tag=18 +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Enumeration)] +pub enum Gender { + Unknown = 0, + Female = 1, + Male = 2, +} +``` + +## FAQ + +1. **Could `prost` be implemented as a serializer for [Serde](https://serde.rs/)?** + + Probably not, however I would like to hear from a Serde expert on the matter. + There are two complications with trying to serialize Protobuf messages with + Serde: + + - Protobuf fields require a numbered tag, and curently there appears to be no + mechanism suitable for this in `serde`. + - The mapping of Protobuf type to Rust type is not 1-to-1. As a result, + trait-based approaches to dispatching don't work very well. Example: six + different Protobuf field types correspond to a Rust `Vec`: `repeated + int32`, `repeated sint32`, `repeated sfixed32`, and their packed + counterparts. + + But it is possible to place `serde` derive tags onto the generated types, so + the same structure can support both `prost` and `Serde`. + +2. **I get errors when trying to run `cargo test` on MacOS** + + If the errors are about missing `autoreconf` or similar, you can probably fix + them by running + + ``` + brew install automake + brew install libtool + ``` + +## License + +`prost` is distributed under the terms of the Apache License (Version 2.0). + +See [LICENSE](LICENSE) for details. + +Copyright 2017 Dan Burkert diff --git a/third_party/prost/benches/varint.rs b/third_party/prost/benches/varint.rs new file mode 100644 index 00000000000..045691e5f31 --- /dev/null +++ b/third_party/prost/benches/varint.rs @@ -0,0 +1,94 @@ +use std::mem; + +use bytes::Buf; +use criterion::{Benchmark, Criterion, Throughput}; +use prost::encoding::{decode_varint, encode_varint, encoded_len_varint}; +use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng}; + +fn benchmark_varint(criterion: &mut Criterion, name: &str, mut values: Vec) { + // Shuffle the values in a stable order. + values.shuffle(&mut StdRng::seed_from_u64(0)); + + let encoded_len = values + .iter() + .cloned() + .map(encoded_len_varint) + .sum::() as u64; + let decoded_len = (values.len() * mem::size_of::()) as u64; + + let encode_values = values.clone(); + let encode = Benchmark::new("encode", move |b| { + let mut buf = Vec::::with_capacity(encode_values.len() * 10); + b.iter(|| { + buf.clear(); + for &value in &encode_values { + encode_varint(value, &mut buf); + } + criterion::black_box(&buf); + }) + }) + .throughput(Throughput::Bytes(encoded_len)); + + let mut decode_values = values.clone(); + let decode = Benchmark::new("decode", move |b| { + let mut buf = Vec::with_capacity(decode_values.len() * 10); + for &value in &decode_values { + encode_varint(value, &mut buf); + } + + b.iter(|| { + decode_values.clear(); + let mut buf = &mut buf.as_slice(); + while buf.has_remaining() { + let value = decode_varint(&mut buf).unwrap(); + decode_values.push(value); + } + criterion::black_box(&decode_values); + }) + }) + .throughput(Throughput::Bytes(decoded_len)); + + let encoded_len = Benchmark::new("encoded_len", move |b| { + b.iter(|| { + let mut sum = 0; + for &value in &values { + sum += encoded_len_varint(value); + } + criterion::black_box(sum); + }) + }) + .throughput(Throughput::Bytes(decoded_len)); + + let name = format!("varint/{}", name); + criterion + .bench(&name, encode) + .bench(&name, decode) + .bench(&name, encoded_len); +} + +fn main() { + let mut criterion = Criterion::default().configure_from_args(); + + // Benchmark encoding and decoding 100 small (1 byte) varints. + benchmark_varint(&mut criterion, "small", (0..100).collect()); + + // Benchmark encoding and decoding 100 medium (5 byte) varints. + benchmark_varint(&mut criterion, "medium", (1 << 28..).take(100).collect()); + + // Benchmark encoding and decoding 100 large (10 byte) varints. + benchmark_varint(&mut criterion, "large", (1 << 63..).take(100).collect()); + + // Benchmark encoding and decoding 100 varints of mixed width (average 5.5 bytes). + benchmark_varint( + &mut criterion, + "mixed", + (0..10) + .flat_map(move |width| { + let exponent = width * 7; + (0..10).map(move |offset| offset + (1 << exponent)) + }) + .collect(), + ); + + criterion.final_summary(); +} diff --git a/third_party/prost/clippy.toml b/third_party/prost/clippy.toml new file mode 100644 index 00000000000..5988e12d8fc --- /dev/null +++ b/third_party/prost/clippy.toml @@ -0,0 +1 @@ +too-many-arguments-threshold=8 diff --git a/third_party/prost/conformance/Cargo.toml b/third_party/prost/conformance/Cargo.toml new file mode 100644 index 00000000000..7783eb5c559 --- /dev/null +++ b/third_party/prost/conformance/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "conformance" +version = "0.0.0" +authors = ["Dan Burkert "] +publish = false +edition = "2018" + +[dependencies] +bytes = "0.5" +env_logger = { version = "0.7", default-features = false } +log = "0.4" +prost = { path = ".." } +protobuf = { path = "../protobuf" } +tests = { path = "../tests" } diff --git a/third_party/prost/conformance/failing_tests.txt b/third_party/prost/conformance/failing_tests.txt new file mode 100644 index 00000000000..73f00caa660 --- /dev/null +++ b/third_party/prost/conformance/failing_tests.txt @@ -0,0 +1,3 @@ +# TODO(danburkert/prost#2): prost doesn't preserve unknown fields. +Required.Proto2.ProtobufInput.UnknownVarint.ProtobufOutput +Required.Proto3.ProtobufInput.UnknownVarint.ProtobufOutput diff --git a/third_party/prost/conformance/src/main.rs b/third_party/prost/conformance/src/main.rs new file mode 100644 index 00000000000..1aee7c35fe4 --- /dev/null +++ b/third_party/prost/conformance/src/main.rs @@ -0,0 +1,115 @@ +use std::io::{self, Read, Write}; + +use bytes::{Buf, BufMut}; +use prost::Message; + +use protobuf::conformance::{ + conformance_request, conformance_response, ConformanceRequest, ConformanceResponse, WireFormat, +}; +use protobuf::test_messages::proto2::TestAllTypesProto2; +use protobuf::test_messages::proto3::TestAllTypesProto3; +use tests::{roundtrip, RoundtripResult}; + +fn main() -> io::Result<()> { + env_logger::init(); + let mut bytes = Vec::new(); + + loop { + bytes.resize(4, 0); + + if io::stdin().read_exact(&mut *bytes).is_err() { + // No more test cases. + return Ok(()); + } + + let len = bytes.as_slice().get_u32_le() as usize; + + bytes.resize(len, 0); + io::stdin().read_exact(&mut *bytes)?; + + let result = match ConformanceRequest::decode(&*bytes) { + Ok(request) => handle_request(request), + Err(error) => conformance_response::Result::ParseError(format!("{:?}", error)), + }; + + let mut response = ConformanceResponse::default(); + response.result = Some(result); + + let len = response.encoded_len(); + bytes.clear(); + bytes.put_u32_le(len as u32); + response.encode(&mut bytes)?; + assert_eq!(len + 4, bytes.len()); + + let mut stdout = io::stdout(); + stdout.lock().write_all(&bytes)?; + stdout.flush()?; + } +} + +fn handle_request(request: ConformanceRequest) -> conformance_response::Result { + match request.requested_output_format() { + WireFormat::Unspecified => { + return conformance_response::Result::ParseError( + "output format unspecified".to_string(), + ); + } + WireFormat::Json => { + return conformance_response::Result::Skipped( + "JSON output is not supported".to_string(), + ); + } + WireFormat::Jspb => { + return conformance_response::Result::Skipped( + "JSPB output is not supported".to_string(), + ); + } + WireFormat::TextFormat => { + return conformance_response::Result::Skipped( + "TEXT_FORMAT output is not supported".to_string(), + ); + } + WireFormat::Protobuf => (), + }; + + let buf = match request.payload { + None => return conformance_response::Result::ParseError("no payload".to_string()), + Some(conformance_request::Payload::JsonPayload(_)) => { + return conformance_response::Result::Skipped( + "JSON input is not supported".to_string(), + ); + } + Some(conformance_request::Payload::JspbPayload(_)) => { + return conformance_response::Result::Skipped( + "JSON input is not supported".to_string(), + ); + } + Some(conformance_request::Payload::TextPayload(_)) => { + return conformance_response::Result::Skipped( + "JSON input is not supported".to_string(), + ); + } + Some(conformance_request::Payload::ProtobufPayload(buf)) => buf, + }; + + let roundtrip = match &*request.message_type { + "protobuf_test_messages.proto2.TestAllTypesProto2" => roundtrip::(&buf), + "protobuf_test_messages.proto3.TestAllTypesProto3" => roundtrip::(&buf), + _ => { + return conformance_response::Result::ParseError(format!( + "unknown message type: {}", + request.message_type + )); + } + }; + + match roundtrip { + RoundtripResult::Ok(buf) => conformance_response::Result::ProtobufPayload(buf), + RoundtripResult::DecodeError(error) => { + conformance_response::Result::ParseError(error.to_string()) + } + RoundtripResult::Error(error) => { + conformance_response::Result::RuntimeError(error.to_string()) + } + } +} diff --git a/third_party/prost/conformance/tests/conformance.rs b/third_party/prost/conformance/tests/conformance.rs new file mode 100644 index 00000000000..39d87ce8aef --- /dev/null +++ b/third_party/prost/conformance/tests/conformance.rs @@ -0,0 +1,33 @@ +#![cfg(not(target_os = "windows"))] + +use std::env; +use std::process::Command; + +use protobuf::conformance; + +/// Runs the protobuf conformance test. This must be done in an integration test +/// so that Cargo will build the proto-conformance binary. +#[test] +fn test_conformance() { + // Get the path to the proto-conformance binary. Adapted from + // https://github.com/rust-lang/cargo/blob/19fdb308cdbb25faf4f1e25a71351d8d603fa447/tests/cargotest/support/mod.rs#L306. + let proto_conformance = env::current_exe() + .map(|mut path| { + path.pop(); + if path.ends_with("deps") { + path.pop(); + } + path.join("conformance") + }) + .unwrap(); + + let status = Command::new(conformance::test_runner()) + .arg("--enforce_recommended") + .arg("--failure_list") + .arg("failing_tests.txt") + .arg(proto_conformance) + .status() + .expect("failed to execute conformance-test-runner"); + + assert!(status.success(), "proto conformance test failed"); +} diff --git a/third_party/prost/fuzz/.gitignore b/third_party/prost/fuzz/.gitignore new file mode 100644 index 00000000000..ff2738685d7 --- /dev/null +++ b/third_party/prost/fuzz/.gitignore @@ -0,0 +1,2 @@ +artifacts +corpus diff --git a/third_party/prost/fuzz/Cargo.toml b/third_party/prost/fuzz/Cargo.toml new file mode 100644 index 00000000000..5a1f425c32a --- /dev/null +++ b/third_party/prost/fuzz/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "fuzz" +version = "0.0.0" +authors = ["Dan Burkert "] +publish = false +edition = "2018" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer-sys.git" } +protobuf = { path = "../protobuf" } +tests = { path = "../tests" } + +[[bin]] +name = "proto3" +path = "fuzzers/proto3.rs" + +[[bin]] +name = "proto2" +path = "fuzzers/proto2.rs" diff --git a/third_party/prost/fuzz/fuzzers/proto2.rs b/third_party/prost/fuzz/fuzzers/proto2.rs new file mode 100644 index 00000000000..9573009ff14 --- /dev/null +++ b/third_party/prost/fuzz/fuzzers/proto2.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use protobuf::test_messages::proto2::TestAllTypesProto2; +use tests::roundtrip; + +fuzz_target!(|data: &[u8]| { + let _ = roundtrip::(data).unwrap_error(); +}); + diff --git a/third_party/prost/fuzz/fuzzers/proto3.rs b/third_party/prost/fuzz/fuzzers/proto3.rs new file mode 100644 index 00000000000..309701636a4 --- /dev/null +++ b/third_party/prost/fuzz/fuzzers/proto3.rs @@ -0,0 +1,9 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use protobuf::test_messages::proto3::TestAllTypesProto3; +use tests::roundtrip; + +fuzz_target!(|data: &[u8]| { + let _ = roundtrip::(data).unwrap_error(); +}); diff --git a/third_party/prost/prepare-release.sh b/third_party/prost/prepare-release.sh new file mode 100755 index 00000000000..34e5202c24f --- /dev/null +++ b/third_party/prost/prepare-release.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Script which automates modifying source version fields, and creating a release +# commit and tag. The commit and tag are not automatically pushed, nor are the +# crates published (see publish-release.sh). + +set -ex + +if [ "$#" -ne 1 ] +then + echo "Usage: $0 " + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +VERSION="$1" +MINOR="$( echo ${VERSION} | cut -d\. -f1-2 )" + +VERSION_MATCHER="([a-z0-9\\.-]+)" +PROST_CRATE_MATCHER="(prost|prost-[a-z]+)" + +# Update the README.md. +sed -i -E "s/${PROST_CRATE_MATCHER} = \"${VERSION_MATCHER}\"/\1 = \"${MINOR}\"/" "$DIR/README.md" + +# Update html_root_url attributes. +sed -i -E "s~html_root_url = \"https://docs\.rs/${PROST_CRATE_MATCHER}/$VERSION_MATCHER\"~html_root_url = \"https://docs.rs/\1/${VERSION}\"~" \ + "$DIR/src/lib.rs" \ + "$DIR/prost-derive/src/lib.rs" \ + "$DIR/prost-build/src/lib.rs" \ + "$DIR/prost-types/src/lib.rs" + +# Update Cargo.toml version fields. +sed -i -E "s/^version = \"${VERSION_MATCHER}\"$/version = \"${VERSION}\"/" \ + "$DIR/Cargo.toml" \ + "$DIR/prost-derive/Cargo.toml" \ + "$DIR/prost-build/Cargo.toml" \ + "$DIR/prost-types/Cargo.toml" + +# Update Cargo.toml dependency versions. +sed -i -E "s/^${PROST_CRATE_MATCHER} = \{ version = \"${VERSION_MATCHER}\"/\1 = { version = \"${VERSION}\"/" \ + "$DIR/Cargo.toml" \ + "$DIR/prost-derive/Cargo.toml" \ + "$DIR/prost-build/Cargo.toml" \ + "$DIR/prost-types/Cargo.toml" + +git commit -a -m "release ${VERSION}" +git tag -a "v${VERSION}" -m "release ${VERSION}" diff --git a/third_party/prost/prost-build/Cargo.toml b/third_party/prost/prost-build/Cargo.toml new file mode 100644 index 00000000000..3f4360720d7 --- /dev/null +++ b/third_party/prost/prost-build/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "prost-build" +version = "0.6.1" +authors = ["Dan Burkert "] +license = "Apache-2.0" +repository = "https://github.com/danburkert/prost" +documentation = "https://docs.rs/prost-build" +readme = "README.md" +description = "A Protocol Buffers implementation for the Rust Language." +edition = "2018" + +[dependencies] +bytes = { version = "0.5", default-features = false } +heck = "0.3" +itertools = "0.9" +log = "0.4" +multimap = { version = "0.8", default-features = false } +petgraph = { version = "0.5", default-features = false } +prost = { version = "0.6.1", path = "..", default-features = false } +prost-types = { version = "0.6.1", path = "../prost-types", default-features = false } +tempfile = "3" + +[build-dependencies] +which = { version = "3", default-features = false } + +[dev-dependencies] +env_logger = { version = "0.7", default-features = false } diff --git a/third_party/prost/prost-build/README.md b/third_party/prost/prost-build/README.md new file mode 100644 index 00000000000..e16957c8e4a --- /dev/null +++ b/third_party/prost/prost-build/README.md @@ -0,0 +1,16 @@ +[![Documentation](https://docs.rs/prost-build/badge.svg)](https://docs.rs/prost-build/) +[![Crate](https://img.shields.io/crates/v/prost-build.svg)](https://crates.io/crates/prost-build) + +# `prost-build` + +`prost-build` makes it easy to generate Rust code from `.proto` files as part of +a Cargo build. See the crate [documentation](https://docs.rs/prost-build/) for examples +of how to integrate `prost-build` into a Cargo project. + +## License + +`prost-build` is distributed under the terms of the Apache License (Version 2.0). + +See [LICENSE](../LICENSE) for details. + +Copyright 2017 Dan Burkert diff --git a/third_party/prost/prost-build/build.rs b/third_party/prost/prost-build/build.rs new file mode 100644 index 00000000000..d5586516e1a --- /dev/null +++ b/third_party/prost/prost-build/build.rs @@ -0,0 +1,104 @@ +//! Finds the appropriate `protoc` binary and Protobuf include directory for this host, and outputs +//! build directives so that the main `prost-build` crate can use them. +//! +//! The following locations are checked for `protoc` in decreasing priority: +//! +//! 1. The `PROTOC` environment variable. +//! 2. The bundled `protoc`. +//! 3. The `protoc` on the `PATH`. +//! +//! If no `protoc` binary is available in these locations, the build fails. +//! +//! The following locations are checked for the Protobuf include directory in decreasing priority: +//! +//! 1. The `PROTOC_INCLUDE` environment variable. +//! 2. The bundled Protobuf include directory. + +use std::env; +use std::path::PathBuf; + +/// Returns the path to the location of the bundled Protobuf artifacts. +fn bundle_path() -> PathBuf { + env::current_dir() + .unwrap() + .join("third-party") + .join("protobuf") +} + +/// Returns the path to the `protoc` pointed to by the `PROTOC` environment variable, if it is set. +fn env_protoc() -> Option { + let protoc = match env::var_os("PROTOC") { + Some(path) => PathBuf::from(path), + None => return None, + }; + + Some(protoc) +} + +/// Returns the path to the bundled `protoc`, if it is available for the host platform. +fn bundled_protoc() -> Option { + let protoc_bin_name = match (env::consts::OS, env::consts::ARCH) { + ("linux", "x86") => "protoc-linux-x86_32", + ("linux", "x86_64") => "protoc-linux-x86_64", + ("linux", "aarch64") => "protoc-linux-aarch_64", + ("macos", "x86_64") => "protoc-osx-x86_64", + ("windows", _) => "protoc-win32.exe", + _ => return None, + }; + + Some(bundle_path().join(protoc_bin_name)) +} + +/// Returns the path to the `protoc` included on the `PATH`, if it exists. +fn path_protoc() -> Option { + which::which("protoc").ok() +} + +/// Returns the path to the Protobuf include directory pointed to by the `PROTOC_INCLUDE` +/// environment variable, if it is set. +fn env_protoc_include() -> Option { + let protoc_include = match env::var_os("PROTOC_INCLUDE") { + Some(path) => PathBuf::from(path), + None => return None, + }; + + if !protoc_include.exists() { + panic!( + "PROTOC_INCLUDE environment variable points to non-existent directory ({:?})", + protoc_include + ); + } + if !protoc_include.is_dir() { + panic!( + "PROTOC_INCLUDE environment variable points to a non-directory file ({:?})", + protoc_include + ); + } + + Some(protoc_include) +} + +/// Returns the path to the bundled Protobuf include directory. +fn bundled_protoc_include() -> PathBuf { + bundle_path().join("include") +} + +fn main() { + let protoc = env_protoc() + .or_else(bundled_protoc) + .or_else(path_protoc) + .expect( + "Failed to find the protoc binary. The PROTOC environment variable is not set, \ + there is no bundled protoc for this platform, and protoc is not in the PATH", + ); + + let protoc_include = env_protoc_include().unwrap_or_else(bundled_protoc_include); + + println!("cargo:rustc-env=PROTOC={}", protoc.display()); + println!( + "cargo:rustc-env=PROTOC_INCLUDE={}", + protoc_include.display() + ); + println!("cargo:rerun-if-env-changed=PROTOC"); + println!("cargo:rerun-if-env-changed=PROTOC_INCLUDE"); +} diff --git a/third_party/prost/prost-build/src/ast.rs b/third_party/prost/prost-build/src/ast.rs new file mode 100644 index 00000000000..cc023e2a6e3 --- /dev/null +++ b/third_party/prost/prost-build/src/ast.rs @@ -0,0 +1,132 @@ +use prost_types::source_code_info::Location; + +/// Comments on a Protobuf item. +#[derive(Debug)] +pub struct Comments { + /// Leading detached blocks of comments. + pub leading_detached: Vec>, + + /// Leading comments. + pub leading: Vec, + + /// Trailing comments. + pub trailing: Vec, +} + +impl Comments { + pub(crate) fn from_location(location: &Location) -> Comments { + fn get_lines(comments: S) -> Vec + where + S: AsRef, + { + comments.as_ref().lines().map(str::to_owned).collect() + } + + let leading_detached = location + .leading_detached_comments + .iter() + .map(get_lines) + .collect(); + let leading = location + .leading_comments + .as_ref() + .map_or(Vec::new(), get_lines); + let trailing = location + .trailing_comments + .as_ref() + .map_or(Vec::new(), get_lines); + Comments { + leading_detached, + leading, + trailing, + } + } + + /// Appends the comments to a buffer with indentation. + /// + /// Each level of indentation corresponds to four space (' ') characters. + pub fn append_with_indent(&self, indent_level: u8, buf: &mut String) { + // Append blocks of detached comments. + for detached_block in &self.leading_detached { + for line in detached_block { + for _ in 0..indent_level { + buf.push_str(" "); + } + buf.push_str("//"); + buf.push_str(line); + buf.push_str("\n"); + } + buf.push_str("\n"); + } + + // Append leading comments. + for line in &self.leading { + for _ in 0..indent_level { + buf.push_str(" "); + } + buf.push_str("///"); + buf.push_str(line); + buf.push_str("\n"); + } + + // Append an empty comment line if there are leading and trailing comments. + if !self.leading.is_empty() && !self.trailing.is_empty() { + for _ in 0..indent_level { + buf.push_str(" "); + } + buf.push_str("///\n"); + } + + // Append trailing comments. + for line in &self.trailing { + for _ in 0..indent_level { + buf.push_str(" "); + } + buf.push_str("///"); + buf.push_str(line); + buf.push_str("\n"); + } + } +} + +/// A service descriptor. +#[derive(Debug)] +pub struct Service { + /// The service name in Rust style. + pub name: String, + /// The service name as it appears in the .proto file. + pub proto_name: String, + /// The package name as it appears in the .proto file. + pub package: String, + /// The service comments. + pub comments: Comments, + /// The service methods. + pub methods: Vec, + /// The service options. + pub options: prost_types::ServiceOptions, +} + +/// A service method descriptor. +#[derive(Debug)] +pub struct Method { + /// The name of the method in Rust style. + pub name: String, + /// The name of the method as it appears in the .proto file. + pub proto_name: String, + /// The method comments. + pub comments: Comments, + /// The input Rust type. + pub input_type: String, + /// The output Rust type. + pub output_type: String, + /// The input Protobuf type. + pub input_proto_type: String, + /// The output Protobuf type. + pub output_proto_type: String, + /// The method options. + pub options: prost_types::MethodOptions, + /// Identifies if client streams multiple client messages. + pub client_streaming: bool, + /// Identifies if server streams multiple server messages. + pub server_streaming: bool, +} diff --git a/third_party/prost/prost-build/src/code_generator.rs b/third_party/prost/prost-build/src/code_generator.rs new file mode 100644 index 00000000000..7323eb613d7 --- /dev/null +++ b/third_party/prost/prost-build/src/code_generator.rs @@ -0,0 +1,1004 @@ +use std::ascii; +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::iter; + +use itertools::{Either, Itertools}; +use log::debug; +use multimap::MultiMap; +use prost_types::field_descriptor_proto::{Label, Type}; +use prost_types::source_code_info::Location; +use prost_types::{ + DescriptorProto, EnumDescriptorProto, EnumValueDescriptorProto, FieldDescriptorProto, + FieldOptions, FileDescriptorProto, OneofDescriptorProto, ServiceDescriptorProto, + SourceCodeInfo, +}; + +use crate::ast::{Comments, Method, Service}; +use crate::extern_paths::ExternPaths; +use crate::ident::{match_ident, to_snake, to_upper_camel}; +use crate::message_graph::MessageGraph; +use crate::Config; + +#[derive(PartialEq)] +enum Syntax { + Proto2, + Proto3, +} + +pub struct CodeGenerator<'a> { + config: &'a mut Config, + package: String, + source_info: SourceCodeInfo, + syntax: Syntax, + message_graph: &'a MessageGraph, + extern_paths: &'a ExternPaths, + depth: u8, + path: Vec, + buf: &'a mut String, +} + +impl<'a> CodeGenerator<'a> { + pub fn generate( + config: &mut Config, + message_graph: &MessageGraph, + extern_paths: &ExternPaths, + file: FileDescriptorProto, + buf: &mut String, + ) { + let mut source_info = file + .source_code_info + .expect("no source code info in request"); + source_info.location.retain(|location| { + let len = location.path.len(); + len > 0 && len % 2 == 0 + }); + source_info + .location + .sort_by_key(|location| location.path.clone()); + + let syntax = match file.syntax.as_ref().map(String::as_str) { + None | Some("proto2") => Syntax::Proto2, + Some("proto3") => Syntax::Proto3, + Some(s) => panic!("unknown syntax: {}", s), + }; + + let mut code_gen = CodeGenerator { + config, + package: file.package.unwrap(), + source_info, + syntax, + message_graph, + extern_paths, + depth: 0, + path: Vec::new(), + buf, + }; + + debug!( + "file: {:?}, package: {:?}", + file.name.as_ref().unwrap(), + code_gen.package + ); + + code_gen.path.push(4); + for (idx, message) in file.message_type.into_iter().enumerate() { + code_gen.path.push(idx as i32); + code_gen.append_message(message); + code_gen.path.pop(); + } + code_gen.path.pop(); + + code_gen.path.push(5); + for (idx, desc) in file.enum_type.into_iter().enumerate() { + code_gen.path.push(idx as i32); + code_gen.append_enum(desc); + code_gen.path.pop(); + } + code_gen.path.pop(); + + if code_gen.config.service_generator.is_some() { + code_gen.path.push(6); + for (idx, service) in file.service.into_iter().enumerate() { + code_gen.path.push(idx as i32); + code_gen.push_service(service); + code_gen.path.pop(); + } + + if let Some(service_generator) = code_gen.config.service_generator.as_mut() { + service_generator.finalize(code_gen.buf); + } + + code_gen.path.pop(); + } + } + + fn append_message(&mut self, message: DescriptorProto) { + debug!(" message: {:?}", message.name()); + + let message_name = message.name().to_string(); + let fq_message_name = format!(".{}.{}", self.package, message.name()); + + // Skip external types. + if self.extern_paths.resolve_ident(&fq_message_name).is_some() { + return; + } + + // Split the nested message types into a vector of normal nested message types, and a map + // of the map field entry types. The path index of the nested message types is preserved so + // that comments can be retrieved. + type NestedTypes = Vec<(DescriptorProto, usize)>; + type MapTypes = HashMap; + let (nested_types, map_types): (NestedTypes, MapTypes) = message + .nested_type + .into_iter() + .enumerate() + .partition_map(|(idx, nested_type)| { + if nested_type + .options + .as_ref() + .and_then(|options| options.map_entry) + .unwrap_or(false) + { + let key = nested_type.field[0].clone(); + let value = nested_type.field[1].clone(); + assert_eq!("key", key.name()); + assert_eq!("value", value.name()); + + let name = format!("{}.{}", &fq_message_name, nested_type.name()); + Either::Right((name, (key, value))) + } else { + Either::Left((nested_type, idx)) + } + }); + + // Split the fields into a vector of the normal fields, and oneof fields. + // Path indexes are preserved so that comments can be retrieved. + type Fields = Vec<(FieldDescriptorProto, usize)>; + type OneofFields = MultiMap; + let (fields, mut oneof_fields): (Fields, OneofFields) = message + .field + .into_iter() + .enumerate() + .partition_map(|(idx, field)| { + if let Some(oneof_index) = field.oneof_index { + Either::Right((oneof_index, (field, idx))) + } else { + Either::Left((field, idx)) + } + }); + + assert_eq!(oneof_fields.len(), message.oneof_decl.len()); + + self.append_doc(); + self.append_type_attributes(&fq_message_name); + self.push_indent(); + self.buf + .push_str("#[derive(Clone, PartialEq, ::prost::Message)]\n"); + self.push_indent(); + self.buf.push_str("pub struct "); + self.buf.push_str(&to_upper_camel(&message_name)); + self.buf.push_str(" {\n"); + + self.depth += 1; + self.path.push(2); + for (field, idx) in fields { + self.path.push(idx as i32); + match field + .type_name + .as_ref() + .and_then(|type_name| map_types.get(type_name)) + { + Some(&(ref key, ref value)) => { + self.append_map_field(&fq_message_name, field, key, value) + } + None => self.append_field(&fq_message_name, field), + } + self.path.pop(); + } + self.path.pop(); + + self.path.push(8); + for (idx, oneof) in message.oneof_decl.iter().enumerate() { + let idx = idx as i32; + self.path.push(idx); + self.append_oneof_field( + &message_name, + &fq_message_name, + oneof, + oneof_fields.get_vec(&idx).unwrap(), + ); + self.path.pop(); + } + self.path.pop(); + + self.depth -= 1; + self.push_indent(); + self.buf.push_str("}\n"); + + if !message.enum_type.is_empty() || !nested_types.is_empty() || !oneof_fields.is_empty() { + self.push_mod(&message_name); + self.path.push(3); + for (nested_type, idx) in nested_types { + self.path.push(idx as i32); + self.append_message(nested_type); + self.path.pop(); + } + self.path.pop(); + + self.path.push(4); + for (idx, nested_enum) in message.enum_type.into_iter().enumerate() { + self.path.push(idx as i32); + self.append_enum(nested_enum); + self.path.pop(); + } + self.path.pop(); + + for (idx, oneof) in message.oneof_decl.into_iter().enumerate() { + let idx = idx as i32; + self.append_oneof( + &fq_message_name, + oneof, + idx, + oneof_fields.remove(&idx).unwrap(), + ); + } + + self.pop_mod(); + } + } + + fn append_type_attributes(&mut self, msg_name: &str) { + assert_eq!(b'.', msg_name.as_bytes()[0]); + // TODO: this clone is dirty, but expedious. + for (matcher, attribute) in self.config.type_attributes.clone() { + if match_ident(&matcher, msg_name, None) { + self.push_indent(); + self.buf.push_str(&attribute); + self.buf.push('\n'); + } + } + } + + fn append_field_attributes(&mut self, msg_name: &str, field_name: &str) { + assert_eq!(b'.', msg_name.as_bytes()[0]); + // TODO: this clone is dirty, but expedious. + for (matcher, attribute) in self.config.field_attributes.clone() { + if match_ident(&matcher, msg_name, Some(field_name)) { + self.push_indent(); + self.buf.push_str(&attribute); + self.buf.push('\n'); + } + } + } + + fn append_field(&mut self, msg_name: &str, field: FieldDescriptorProto) { + let type_ = field.r#type(); + let repeated = field.label == Some(Label::Repeated as i32); + let deprecated = self.deprecated(&field); + let optional = self.optional(&field); + let ty = self.resolve_type(&field); + + let boxed = !repeated + && (type_ == Type::Message || type_ == Type::Group) + && self.message_graph.is_nested(field.type_name(), msg_name); + + debug!( + " field: {:?}, type: {:?}, boxed: {}", + field.name(), + ty, + boxed + ); + + self.append_doc(); + + if deprecated { + self.push_indent(); + self.buf.push_str("#[deprecated]\n"); + } + + self.push_indent(); + self.buf.push_str("#[prost("); + let type_tag = self.field_type_tag(&field); + self.buf.push_str(&type_tag); + + match field.label() { + Label::Optional => { + if optional { + self.buf.push_str(", optional"); + } + } + Label::Required => self.buf.push_str(", required"), + Label::Repeated => { + self.buf.push_str(", repeated"); + if can_pack(&field) + && !field + .options + .as_ref() + .map_or(self.syntax == Syntax::Proto3, |options| options.packed()) + { + self.buf.push_str(", packed=\"false\""); + } + } + } + + if boxed { + self.buf.push_str(", boxed"); + } + self.buf.push_str(", tag=\""); + self.buf.push_str(&field.number().to_string()); + + if let Some(ref default) = field.default_value { + self.buf.push_str("\", default=\""); + if type_ == Type::Bytes { + self.buf.push_str("b\\\""); + for b in unescape_c_escape_string(default) { + self.buf.extend( + ascii::escape_default(b).flat_map(|c| (c as char).escape_default()), + ); + } + self.buf.push_str("\\\""); + } else if type_ == Type::Enum { + let enum_value = to_upper_camel(default); + let stripped_prefix = if self.config.strip_enum_prefix { + // Field types are fully qualified, so we extract + // the last segment and strip it from the left + // side of the default value. + let enum_type = field + .type_name + .as_ref() + .and_then(|ty| ty.split('.').last()) + .unwrap(); + + strip_enum_prefix(&to_upper_camel(&enum_type), &enum_value) + } else { + &enum_value + }; + self.buf.push_str(stripped_prefix); + } else { + // TODO: this is only correct if the Protobuf escaping matches Rust escaping. To be + // safer, we should unescape the Protobuf string and re-escape it with the Rust + // escaping mechanisms. + self.buf.push_str(default); + } + } + + self.buf.push_str("\")]\n"); + self.append_field_attributes(msg_name, field.name()); + self.push_indent(); + self.buf.push_str("pub "); + self.buf.push_str(&to_snake(field.name())); + self.buf.push_str(": "); + if repeated { + self.buf.push_str("::prost::alloc::vec::Vec<"); + } else if optional { + self.buf.push_str("::core::option::Option<"); + } + if boxed { + self.buf.push_str("::prost::alloc::boxed::Box<"); + } + self.buf.push_str(&ty); + if boxed { + self.buf.push_str(">"); + } + if repeated || optional { + self.buf.push_str(">"); + } + self.buf.push_str(",\n"); + } + + fn append_map_field( + &mut self, + msg_name: &str, + field: FieldDescriptorProto, + key: &FieldDescriptorProto, + value: &FieldDescriptorProto, + ) { + let key_ty = self.resolve_type(key); + let value_ty = self.resolve_type(value); + + debug!( + " map field: {:?}, key type: {:?}, value type: {:?}", + field.name(), + key_ty, + value_ty + ); + + self.append_doc(); + self.push_indent(); + + let btree_map = self + .config + .btree_map + .iter() + .any(|matcher| match_ident(matcher, msg_name, Some(field.name()))); + let (annotation_ty, lib_name, rust_ty) = if btree_map { + ("btree_map", "::prost::alloc::collections", "BTreeMap") + } else { + ("map", "::std::collections", "HashMap") + }; + + let key_tag = self.field_type_tag(key); + let value_tag = self.map_value_type_tag(value); + self.buf.push_str(&format!( + "#[prost({}=\"{}, {}\", tag=\"{}\")]\n", + annotation_ty, + key_tag, + value_tag, + field.number() + )); + self.append_field_attributes(msg_name, field.name()); + self.push_indent(); + self.buf.push_str(&format!( + "pub {}: {}::{}<{}, {}>,\n", + to_snake(field.name()), + lib_name, + rust_ty, + key_ty, + value_ty + )); + } + + fn append_oneof_field( + &mut self, + message_name: &str, + fq_message_name: &str, + oneof: &OneofDescriptorProto, + fields: &[(FieldDescriptorProto, usize)], + ) { + let name = format!( + "{}::{}", + to_snake(message_name), + to_upper_camel(oneof.name()) + ); + self.append_doc(); + self.push_indent(); + self.buf.push_str(&format!( + "#[prost(oneof=\"{}\", tags=\"{}\")]\n", + name, + fields + .iter() + .map(|&(ref field, _)| field.number()) + .join(", ") + )); + self.append_field_attributes(fq_message_name, oneof.name()); + self.push_indent(); + self.buf.push_str(&format!( + "pub {}: ::core::option::Option<{}>,\n", + to_snake(oneof.name()), + name + )); + } + + fn append_oneof( + &mut self, + msg_name: &str, + oneof: OneofDescriptorProto, + idx: i32, + fields: Vec<(FieldDescriptorProto, usize)>, + ) { + self.path.push(8); + self.path.push(idx); + self.append_doc(); + self.path.pop(); + self.path.pop(); + + let oneof_name = format!("{}.{}", msg_name, oneof.name()); + self.append_type_attributes(&oneof_name); + self.push_indent(); + self.buf + .push_str("#[derive(Clone, PartialEq, ::prost::Oneof)]\n"); + self.push_indent(); + self.buf.push_str("pub enum "); + self.buf.push_str(&to_upper_camel(oneof.name())); + self.buf.push_str(" {\n"); + + self.path.push(2); + self.depth += 1; + for (field, idx) in fields { + let type_ = field.r#type(); + + self.path.push(idx as i32); + self.append_doc(); + self.path.pop(); + + self.push_indent(); + let ty_tag = self.field_type_tag(&field); + self.buf.push_str(&format!( + "#[prost({}, tag=\"{}\")]\n", + ty_tag, + field.number() + )); + self.append_field_attributes(&oneof_name, field.name()); + + self.push_indent(); + let ty = self.resolve_type(&field); + + let boxed = (type_ == Type::Message || type_ == Type::Group) + && self.message_graph.is_nested(field.type_name(), msg_name); + + debug!( + " oneof: {:?}, type: {:?}, boxed: {}", + field.name(), + ty, + boxed + ); + + if boxed { + self.buf.push_str(&format!( + "{}(::prost::alloc::boxed::Box<{}>),\n", + to_upper_camel(field.name()), + ty + )); + } else { + self.buf + .push_str(&format!("{}({}),\n", to_upper_camel(field.name()), ty)); + } + } + self.depth -= 1; + self.path.pop(); + + self.push_indent(); + self.buf.push_str("}\n"); + } + + fn location(&self) -> &Location { + let idx = self + .source_info + .location + .binary_search_by_key(&&self.path[..], |location| &location.path[..]) + .unwrap(); + + &self.source_info.location[idx] + } + + fn append_doc(&mut self) { + Comments::from_location(self.location()).append_with_indent(self.depth, &mut self.buf); + } + + fn append_enum(&mut self, desc: EnumDescriptorProto) { + debug!(" enum: {:?}", desc.name()); + + // Skip external types. + let enum_name = &desc.name(); + let enum_values = &desc.value; + let fq_enum_name = format!(".{}.{}", self.package, enum_name); + if self.extern_paths.resolve_ident(&fq_enum_name).is_some() { + return; + } + + self.append_doc(); + self.append_type_attributes(&fq_enum_name); + self.push_indent(); + self.buf.push_str( + "#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]\n", + ); + self.push_indent(); + self.buf.push_str("#[repr(i32)]\n"); + self.push_indent(); + self.buf.push_str("pub enum "); + self.buf.push_str(&to_upper_camel(desc.name())); + self.buf.push_str(" {\n"); + + let mut numbers = HashSet::new(); + + self.depth += 1; + self.path.push(2); + for (idx, value) in enum_values.iter().enumerate() { + // Skip duplicate enum values. Protobuf allows this when the + // 'allow_alias' option is set. + if !numbers.insert(value.number()) { + continue; + } + + self.path.push(idx as i32); + let stripped_prefix = if self.config.strip_enum_prefix { + Some(to_upper_camel(&enum_name)) + } else { + None + }; + self.append_enum_value(&fq_enum_name, value, stripped_prefix); + self.path.pop(); + } + self.path.pop(); + self.depth -= 1; + + self.push_indent(); + self.buf.push_str("}\n"); + } + + fn append_enum_value( + &mut self, + fq_enum_name: &str, + value: &EnumValueDescriptorProto, + prefix_to_strip: Option, + ) { + self.append_doc(); + self.append_field_attributes(fq_enum_name, &value.name()); + self.push_indent(); + let name = to_upper_camel(value.name()); + let name_unprefixed = match prefix_to_strip { + Some(prefix) => strip_enum_prefix(&prefix, &name), + None => &name, + }; + self.buf.push_str(name_unprefixed); + self.buf.push_str(" = "); + self.buf.push_str(&value.number().to_string()); + self.buf.push_str(",\n"); + } + + fn push_service(&mut self, service: ServiceDescriptorProto) { + let name = service.name().to_owned(); + debug!(" service: {:?}", name); + + let comments = Comments::from_location(self.location()); + + self.path.push(2); + let methods = service + .method + .into_iter() + .enumerate() + .map(|(idx, mut method)| { + debug!(" method: {:?}", method.name()); + self.path.push(idx as i32); + let comments = Comments::from_location(self.location()); + self.path.pop(); + + let name = method.name.take().unwrap(); + let input_proto_type = method.input_type.take().unwrap(); + let output_proto_type = method.output_type.take().unwrap(); + let input_type = self.resolve_ident(&input_proto_type); + let output_type = self.resolve_ident(&output_proto_type); + let client_streaming = method.client_streaming(); + let server_streaming = method.server_streaming(); + + Method { + name: to_snake(&name), + proto_name: name, + comments, + input_type, + output_type, + input_proto_type, + output_proto_type, + options: method.options.unwrap_or_default(), + client_streaming, + server_streaming, + } + }) + .collect(); + self.path.pop(); + + let service = Service { + name: to_upper_camel(&name), + proto_name: name, + package: self.package.clone(), + comments, + methods, + options: service.options.unwrap_or_default(), + }; + + if let Some(service_generator) = self.config.service_generator.as_mut() { + service_generator.generate(service, &mut self.buf) + } + } + + fn push_indent(&mut self) { + for _ in 0..self.depth { + self.buf.push_str(" "); + } + } + + fn push_mod(&mut self, module: &str) { + self.push_indent(); + self.buf.push_str("/// Nested message and enum types in `"); + self.buf.push_str(module); + self.buf.push_str("`.\n"); + + self.push_indent(); + self.buf.push_str("pub mod "); + self.buf.push_str(&to_snake(module)); + self.buf.push_str(" {\n"); + + self.package.push_str("."); + self.package.push_str(module); + + self.depth += 1; + } + + fn pop_mod(&mut self) { + self.depth -= 1; + + let idx = self.package.rfind('.').unwrap(); + self.package.truncate(idx); + + self.push_indent(); + self.buf.push_str("}\n"); + } + + fn resolve_type(&self, field: &FieldDescriptorProto) -> String { + match field.r#type() { + Type::Float => String::from("f32"), + Type::Double => String::from("f64"), + Type::Uint32 | Type::Fixed32 => String::from("u32"), + Type::Uint64 | Type::Fixed64 => String::from("u64"), + Type::Int32 | Type::Sfixed32 | Type::Sint32 | Type::Enum => String::from("i32"), + Type::Int64 | Type::Sfixed64 | Type::Sint64 => String::from("i64"), + Type::Bool => String::from("bool"), + Type::String => String::from("::prost::alloc::string::String"), + Type::Bytes => String::from("::prost::alloc::vec::Vec"), + Type::Group | Type::Message => self.resolve_ident(field.type_name()), + } + } + + fn resolve_ident(&self, pb_ident: &str) -> String { + // protoc should always give fully qualified identifiers. + assert_eq!(".", &pb_ident[..1]); + + if let Some(proto_ident) = self.extern_paths.resolve_ident(pb_ident) { + return proto_ident; + } + + let mut local_path = self.package.split('.').peekable(); + + let mut ident_path = pb_ident[1..].split('.'); + let ident_type = ident_path.next_back().unwrap(); + let mut ident_path = ident_path.peekable(); + + // Skip path elements in common. + while local_path.peek().is_some() && local_path.peek() == ident_path.peek() { + local_path.next(); + ident_path.next(); + } + + local_path + .map(|_| "super".to_string()) + .chain(ident_path.map(to_snake)) + .chain(iter::once(to_upper_camel(ident_type))) + .join("::") + } + + fn field_type_tag(&self, field: &FieldDescriptorProto) -> Cow<'static, str> { + match field.r#type() { + Type::Float => Cow::Borrowed("float"), + Type::Double => Cow::Borrowed("double"), + Type::Int32 => Cow::Borrowed("int32"), + Type::Int64 => Cow::Borrowed("int64"), + Type::Uint32 => Cow::Borrowed("uint32"), + Type::Uint64 => Cow::Borrowed("uint64"), + Type::Sint32 => Cow::Borrowed("sint32"), + Type::Sint64 => Cow::Borrowed("sint64"), + Type::Fixed32 => Cow::Borrowed("fixed32"), + Type::Fixed64 => Cow::Borrowed("fixed64"), + Type::Sfixed32 => Cow::Borrowed("sfixed32"), + Type::Sfixed64 => Cow::Borrowed("sfixed64"), + Type::Bool => Cow::Borrowed("bool"), + Type::String => Cow::Borrowed("string"), + Type::Bytes => Cow::Borrowed("bytes"), + Type::Group => Cow::Borrowed("group"), + Type::Message => Cow::Borrowed("message"), + Type::Enum => Cow::Owned(format!( + "enumeration={:?}", + self.resolve_ident(field.type_name()) + )), + } + } + + fn map_value_type_tag(&self, field: &FieldDescriptorProto) -> Cow<'static, str> { + match field.r#type() { + Type::Enum => Cow::Owned(format!( + "enumeration({})", + self.resolve_ident(field.type_name()) + )), + _ => self.field_type_tag(field), + } + } + + fn optional(&self, field: &FieldDescriptorProto) -> bool { + if field.label() != Label::Optional { + return false; + } + + match field.r#type() { + Type::Message => true, + _ => self.syntax == Syntax::Proto2, + } + } + + /// Returns `true` if the field options includes the `deprecated` option. + fn deprecated(&self, field: &FieldDescriptorProto) -> bool { + field + .options + .as_ref() + .map_or(false, FieldOptions::deprecated) + } +} + +/// Returns `true` if the repeated field type can be packed. +fn can_pack(field: &FieldDescriptorProto) -> bool { + match field.r#type() { + Type::Float + | Type::Double + | Type::Int32 + | Type::Int64 + | Type::Uint32 + | Type::Uint64 + | Type::Sint32 + | Type::Sint64 + | Type::Fixed32 + | Type::Fixed64 + | Type::Sfixed32 + | Type::Sfixed64 + | Type::Bool + | Type::Enum => true, + _ => false, + } +} + +/// Based on [`google::protobuf::UnescapeCEscapeString`][1] +/// [1]: https://github.com/google/protobuf/blob/3.3.x/src/google/protobuf/stubs/strutil.cc#L312-L322 +fn unescape_c_escape_string(s: &str) -> Vec { + let src = s.as_bytes(); + let len = src.len(); + let mut dst = Vec::new(); + + let mut p = 0; + + while p < len { + if src[p] != b'\\' { + dst.push(src[p]); + p += 1; + } else { + p += 1; + if p == len { + panic!( + "invalid c-escaped default binary value ({}): ends with '\'", + s + ) + } + match src[p] { + b'a' => { + dst.push(0x07); + p += 1; + } + b'b' => { + dst.push(0x08); + p += 1; + } + b'f' => { + dst.push(0x0C); + p += 1; + } + b'n' => { + dst.push(0x0A); + p += 1; + } + b'r' => { + dst.push(0x0D); + p += 1; + } + b't' => { + dst.push(0x09); + p += 1; + } + b'v' => { + dst.push(0x0B); + p += 1; + } + b'\\' => { + dst.push(0x5C); + p += 1; + } + b'?' => { + dst.push(0x3F); + p += 1; + } + b'\'' => { + dst.push(0x27); + p += 1; + } + b'"' => { + dst.push(0x22); + p += 1; + } + b'0'..=b'7' => { + eprintln!("another octal: {}, offset: {}", s, &s[p..]); + let mut octal = 0; + for _ in 0..3 { + if p < len && src[p] >= b'0' && src[p] <= b'7' { + eprintln!("\toctal: {}", octal); + octal = octal * 8 + (src[p] - b'0'); + p += 1; + } else { + break; + } + } + dst.push(octal); + } + b'x' | b'X' => { + if p + 2 > len { + panic!( + "invalid c-escaped default binary value ({}): incomplete hex value", + s + ) + } + match u8::from_str_radix(&s[p + 1..p + 3], 16) { + Ok(b) => dst.push(b), + _ => panic!( + "invalid c-escaped default binary value ({}): invalid hex value", + &s[p..p + 2] + ), + } + p += 3; + } + _ => panic!( + "invalid c-escaped default binary value ({}): invalid escape", + s + ), + } + } + } + dst +} + +/// Strip an enum's type name from the prefix of an enum value. +/// +/// This function assumes that both have been formatted to Rust's +/// upper camel case naming conventions. +/// +/// It also tries to handle cases where the stripped name would be +/// invalid - for example, if it were to begin with a number. +fn strip_enum_prefix<'a>(prefix: &str, name: &'a str) -> &'a str { + let stripped = if name.starts_with(prefix) { + &name[prefix.len()..] + } else { + name + }; + // If the next character after the stripped prefix is not + // uppercase, then it means that we didn't have a true prefix - + // for example, "Foo" should not be stripped from "Foobar". + if stripped + .chars() + .next() + .map(char::is_uppercase) + .unwrap_or(false) + { + stripped + } else { + name + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_unescape_c_escape_string() { + assert_eq!( + &b"hello world"[..], + &unescape_c_escape_string("hello world")[..] + ); + + assert_eq!(&b"\0"[..], &unescape_c_escape_string(r#"\0"#)[..]); + + assert_eq!( + &[0o012, 0o156], + &unescape_c_escape_string(r#"\012\156"#)[..] + ); + assert_eq!(&[0x01, 0x02], &unescape_c_escape_string(r#"\x01\x02"#)[..]); + + assert_eq!( + &b"\0\x01\x07\x08\x0C\n\r\t\x0B\\\'\"\xFE"[..], + &unescape_c_escape_string(r#"\0\001\a\b\f\n\r\t\v\\\'\"\xfe"#)[..] + ); + } + + #[test] + fn test_strip_enum_prefix() { + assert_eq!(strip_enum_prefix("Foo", "FooBar"), "Bar"); + assert_eq!(strip_enum_prefix("Foo", "Foobar"), "Foobar"); + assert_eq!(strip_enum_prefix("Foo", "Foo"), "Foo"); + assert_eq!(strip_enum_prefix("Foo", "Bar"), "Bar"); + assert_eq!(strip_enum_prefix("Foo", "Foo1"), "Foo1"); + } +} diff --git a/third_party/prost/prost-build/src/extern_paths.rs b/third_party/prost/prost-build/src/extern_paths.rs new file mode 100644 index 00000000000..2491c6fbc1d --- /dev/null +++ b/third_party/prost/prost-build/src/extern_paths.rs @@ -0,0 +1,170 @@ +use std::collections::{hash_map, HashMap}; + +use itertools::Itertools; + +use crate::ident::{to_snake, to_upper_camel}; + +fn validate_proto_path(path: &str) -> Result<(), String> { + if path.chars().next().map(|c| c != '.').unwrap_or(true) { + return Err(format!( + "Protobuf paths must be fully qualified (begin with a leading '.'): {}", + path + )); + } + if path.split('.').skip(1).any(str::is_empty) { + return Err(format!("invalid fully-qualified Protobuf path: {}", path)); + } + Ok(()) +} + +#[derive(Debug)] +pub struct ExternPaths { + extern_paths: HashMap, +} + +impl ExternPaths { + pub fn new(paths: &[(String, String)], prost_types: bool) -> Result { + let mut extern_paths = ExternPaths { + extern_paths: HashMap::new(), + }; + + for (proto_path, rust_path) in paths { + extern_paths.insert(proto_path.clone(), rust_path.clone())?; + } + + if prost_types { + extern_paths.insert(".google.protobuf".to_string(), "::prost_types".to_string())?; + extern_paths.insert(".google.protobuf.BoolValue".to_string(), "bool".to_string())?; + extern_paths.insert( + ".google.protobuf.BytesValue".to_string(), + "::std::vec::Vec".to_string(), + )?; + extern_paths.insert( + ".google.protobuf.DoubleValue".to_string(), + "f64".to_string(), + )?; + extern_paths.insert(".google.protobuf.Empty".to_string(), "()".to_string())?; + extern_paths.insert(".google.protobuf.FloatValue".to_string(), "f32".to_string())?; + extern_paths.insert(".google.protobuf.Int32Value".to_string(), "i32".to_string())?; + extern_paths.insert(".google.protobuf.Int64Value".to_string(), "i64".to_string())?; + extern_paths.insert( + ".google.protobuf.StringValue".to_string(), + "::std::string::String".to_string(), + )?; + extern_paths.insert( + ".google.protobuf.UInt32Value".to_string(), + "u32".to_string(), + )?; + extern_paths.insert( + ".google.protobuf.UInt64Value".to_string(), + "u64".to_string(), + )?; + } + + Ok(extern_paths) + } + + fn insert(&mut self, proto_path: String, rust_path: String) -> Result<(), String> { + validate_proto_path(&proto_path)?; + match self.extern_paths.entry(proto_path) { + hash_map::Entry::Occupied(occupied) => { + return Err(format!( + "duplicate extern Protobuf path: {}", + occupied.key() + )); + } + hash_map::Entry::Vacant(vacant) => vacant.insert(rust_path), + }; + Ok(()) + } + + pub fn resolve_ident(&self, pb_ident: &str) -> Option { + // protoc should always give fully qualified identifiers. + assert_eq!(".", &pb_ident[..1]); + + if let Some(rust_path) = self.extern_paths.get(pb_ident) { + return Some(rust_path.clone()); + } + + // TODO(danburkert): there must be a more efficient way to do this, maybe a trie? + for (idx, _) in pb_ident.rmatch_indices('.') { + if let Some(rust_path) = self.extern_paths.get(&pb_ident[..idx]) { + let mut segments = pb_ident[idx + 1..].split('.'); + let ident_type = segments.next_back().map(|segment| to_upper_camel(&segment)); + + return Some( + rust_path + .split("::") + .chain(segments) + .enumerate() + .map(|(idx, segment)| { + if idx == 0 && segment == "crate" { + // If the first segment of the path is 'crate', then do not escape + // it into a raw identifier, since it's being used as the keyword. + segment.to_owned() + } else { + to_snake(&segment) + } + }) + .chain(ident_type.into_iter()) + .join("::"), + ); + } + } + + None + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_extern_paths() { + let paths = ExternPaths::new( + &[ + (".foo".to_string(), "::foo1".to_string()), + (".foo.bar".to_string(), "::foo2".to_string()), + (".foo.baz".to_string(), "::foo3".to_string()), + (".foo.Fuzz".to_string(), "::foo4::Fuzz".to_string()), + (".a.b.c.d.e.f".to_string(), "::abc::def".to_string()), + ], + false, + ) + .unwrap(); + + let case = |proto_ident: &str, resolved_ident: &str| { + assert_eq!(paths.resolve_ident(proto_ident).unwrap(), resolved_ident); + }; + + case(".foo", "::foo1"); + case(".foo.Foo", "::foo1::Foo"); + case(".foo.bar", "::foo2"); + case(".foo.Bas", "::foo1::Bas"); + + case(".foo.bar.Bar", "::foo2::Bar"); + case(".foo.Fuzz.Bar", "::foo4::fuzz::Bar"); + + case(".a.b.c.d.e.f", "::abc::def"); + case(".a.b.c.d.e.f.g.FooBar.Baz", "::abc::def::g::foo_bar::Baz"); + + assert!(paths.resolve_ident(".a").is_none()); + assert!(paths.resolve_ident(".a.b").is_none()); + assert!(paths.resolve_ident(".a.c").is_none()); + } + + #[test] + fn test_well_known_types() { + let paths = ExternPaths::new(&[], true).unwrap(); + + let case = |proto_ident: &str, resolved_ident: &str| { + assert_eq!(paths.resolve_ident(proto_ident).unwrap(), resolved_ident); + }; + + case(".google.protobuf.Value", "::prost_types::Value"); + case(".google.protobuf.Duration", "::prost_types::Duration"); + case(".google.protobuf.Empty", "()"); + } +} diff --git a/third_party/prost/prost-build/src/goodbye.proto b/third_party/prost/prost-build/src/goodbye.proto new file mode 100644 index 00000000000..4527d7d1520 --- /dev/null +++ b/third_party/prost/prost-build/src/goodbye.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +import "types.proto"; + +package helloworld; + +service Farewell { + rpc Goodbye (Message) returns (Response) {} +} diff --git a/third_party/prost/prost-build/src/hello.proto b/third_party/prost/prost-build/src/hello.proto new file mode 100644 index 00000000000..8661cc3edd0 --- /dev/null +++ b/third_party/prost/prost-build/src/hello.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +import "types.proto"; + +package helloworld; + +service Greeting { + rpc Hello (Message) returns (Response) {} +} diff --git a/third_party/prost/prost-build/src/ident.rs b/third_party/prost/prost-build/src/ident.rs new file mode 100644 index 00000000000..046fb1efb4f --- /dev/null +++ b/third_party/prost/prost-build/src/ident.rs @@ -0,0 +1,236 @@ +//! Utility functions for working with identifiers. + +use heck::{CamelCase, SnakeCase}; + +/// Converts a `camelCase` or `SCREAMING_SNAKE_CASE` identifier to a `lower_snake` case Rust field +/// identifier. +pub fn to_snake(s: &str) -> String { + let mut ident = s.to_snake_case(); + + // Use a raw identifier if the identifier matches a Rust keyword: + // https://doc.rust-lang.org/reference/keywords.html. + match ident.as_str() { + // 2015 strict keywords. + | "as" | "break" | "const" | "continue" | "else" | "enum" | "false" + | "fn" | "for" | "if" | "impl" | "in" | "let" | "loop" | "match" | "mod" | "move" | "mut" + | "pub" | "ref" | "return" | "static" | "struct" | "trait" | "true" + | "type" | "unsafe" | "use" | "where" | "while" + // 2018 strict keywords. + | "dyn" + // 2015 reserved keywords. + | "abstract" | "become" | "box" | "do" | "final" | "macro" | "override" | "priv" | "typeof" + | "unsized" | "virtual" | "yield" + // 2018 reserved keywords. + | "async" | "await" | "try" => ident.insert_str(0, "r#"), + // the following keywords are not supported as raw identifiers and are therefore suffixed with an underscore. + "self" | "super" | "extern" | "crate" => ident += "_", + _ => (), + } + ident +} + +/// Converts a `snake_case` identifier to an `UpperCamel` case Rust type identifier. +pub fn to_upper_camel(s: &str) -> String { + let mut ident = s.to_camel_case(); + + // Suffix an underscore for the `Self` Rust keyword as it is not allowed as raw identifier. + if ident == "Self" { + ident += "_"; + } + ident +} + +/// Matches a 'matcher' against a fully qualified identifier. +pub fn match_ident(matcher: &str, msg: &str, field: Option<&str>) -> bool { + assert_eq!(b'.', msg.as_bytes()[0]); + + if matcher.is_empty() { + return false; + } else if matcher == "." { + return true; + } + + let match_paths = matcher.split('.').collect::>(); + let field_paths = { + let mut paths = msg.split('.').collect::>(); + if let Some(field) = field { + paths.push(field); + } + paths + }; + + if &matcher[..1] == "." { + // Prefix match. + if match_paths.len() > field_paths.len() { + false + } else { + match_paths[..] == field_paths[..match_paths.len()] + } + // Suffix match. + } else if match_paths.len() > field_paths.len() { + false + } else { + match_paths[..] == field_paths[field_paths.len() - match_paths.len()..] + } +} + +#[cfg(test)] +mod tests { + + #![allow(clippy::cognitive_complexity)] + + use super::*; + + #[test] + fn test_to_snake() { + assert_eq!("foo_bar", &to_snake("FooBar")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("foo_bar_baz", &to_snake("FooBarBAZ")); + assert_eq!("xml_http_request", &to_snake("XMLHttpRequest")); + assert_eq!("r#while", &to_snake("While")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_BUSTER")); + assert_eq!("foo_bar_baz", &to_snake("foo_bar_baz")); + assert_eq!("fuzz_buster", &to_snake("FUZZ_buster")); + assert_eq!("fuzz", &to_snake("_FUZZ")); + assert_eq!("fuzz", &to_snake("_fuzz")); + assert_eq!("fuzz", &to_snake("_Fuzz")); + assert_eq!("fuzz", &to_snake("FUZZ_")); + assert_eq!("fuzz", &to_snake("fuzz_")); + assert_eq!("fuzz", &to_snake("Fuzz_")); + assert_eq!("fuz_z", &to_snake("FuzZ_")); + + // From test_messages_proto3.proto. + assert_eq!("fieldname1", &to_snake("fieldname1")); + assert_eq!("field_name2", &to_snake("field_name2")); + assert_eq!("field_name3", &to_snake("_field_name3")); + assert_eq!("field_name4", &to_snake("field__name4_")); + assert_eq!("field0name5", &to_snake("field0name5")); + assert_eq!("field_0_name6", &to_snake("field_0_name6")); + assert_eq!("field_name7", &to_snake("fieldName7")); + assert_eq!("field_name8", &to_snake("FieldName8")); + assert_eq!("field_name9", &to_snake("field_Name9")); + assert_eq!("field_name10", &to_snake("Field_Name10")); + + // TODO(withoutboats/heck#3) + //assert_eq!("field_name11", &to_snake("FIELD_NAME11")); + assert_eq!("field_name12", &to_snake("FIELD_name12")); + assert_eq!("field_name13", &to_snake("__field_name13")); + assert_eq!("field_name14", &to_snake("__Field_name14")); + assert_eq!("field_name15", &to_snake("field__name15")); + assert_eq!("field_name16", &to_snake("field__Name16")); + assert_eq!("field_name17", &to_snake("field_name17__")); + assert_eq!("field_name18", &to_snake("Field_name18__")); + } + + #[test] + fn test_to_snake_raw_keyword() { + assert_eq!("r#as", &to_snake("as")); + assert_eq!("r#break", &to_snake("break")); + assert_eq!("r#const", &to_snake("const")); + assert_eq!("r#continue", &to_snake("continue")); + assert_eq!("r#else", &to_snake("else")); + assert_eq!("r#enum", &to_snake("enum")); + assert_eq!("r#false", &to_snake("false")); + assert_eq!("r#fn", &to_snake("fn")); + assert_eq!("r#for", &to_snake("for")); + assert_eq!("r#if", &to_snake("if")); + assert_eq!("r#impl", &to_snake("impl")); + assert_eq!("r#in", &to_snake("in")); + assert_eq!("r#let", &to_snake("let")); + assert_eq!("r#loop", &to_snake("loop")); + assert_eq!("r#match", &to_snake("match")); + assert_eq!("r#mod", &to_snake("mod")); + assert_eq!("r#move", &to_snake("move")); + assert_eq!("r#mut", &to_snake("mut")); + assert_eq!("r#pub", &to_snake("pub")); + assert_eq!("r#ref", &to_snake("ref")); + assert_eq!("r#return", &to_snake("return")); + assert_eq!("r#static", &to_snake("static")); + assert_eq!("r#struct", &to_snake("struct")); + assert_eq!("r#trait", &to_snake("trait")); + assert_eq!("r#true", &to_snake("true")); + assert_eq!("r#type", &to_snake("type")); + assert_eq!("r#unsafe", &to_snake("unsafe")); + assert_eq!("r#use", &to_snake("use")); + assert_eq!("r#where", &to_snake("where")); + assert_eq!("r#while", &to_snake("while")); + assert_eq!("r#dyn", &to_snake("dyn")); + assert_eq!("r#abstract", &to_snake("abstract")); + assert_eq!("r#become", &to_snake("become")); + assert_eq!("r#box", &to_snake("box")); + assert_eq!("r#do", &to_snake("do")); + assert_eq!("r#final", &to_snake("final")); + assert_eq!("r#macro", &to_snake("macro")); + assert_eq!("r#override", &to_snake("override")); + assert_eq!("r#priv", &to_snake("priv")); + assert_eq!("r#typeof", &to_snake("typeof")); + assert_eq!("r#unsized", &to_snake("unsized")); + assert_eq!("r#virtual", &to_snake("virtual")); + assert_eq!("r#yield", &to_snake("yield")); + assert_eq!("r#async", &to_snake("async")); + assert_eq!("r#await", &to_snake("await")); + assert_eq!("r#try", &to_snake("try")); + } + + #[test] + fn test_to_snake_non_raw_keyword() { + assert_eq!("self_", &to_snake("self")); + assert_eq!("super_", &to_snake("super")); + assert_eq!("extern_", &to_snake("extern")); + assert_eq!("crate_", &to_snake("crate")); + } + + #[test] + fn test_to_upper_camel() { + assert_eq!("", &to_upper_camel("")); + assert_eq!("F", &to_upper_camel("F")); + assert_eq!("Foo", &to_upper_camel("FOO")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR")); + assert_eq!("FooBar", &to_upper_camel("FOO_BAR_")); + assert_eq!("FooBar", &to_upper_camel("_FOO_BAR_")); + assert_eq!("FuzzBuster", &to_upper_camel("fuzzBuster")); + assert_eq!("FuzzBuster", &to_upper_camel("FuzzBuster")); + assert_eq!("Self_", &to_upper_camel("self")); + } + + #[test] + fn test_match_ident() { + // Prefix matches + assert!(match_ident(".", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident(".foo", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident(".foo.bar", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident(".foo.bar.Baz", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident( + ".foo.bar.Baz.buzz", + ".foo.bar.Baz", + Some("buzz") + )); + + assert!(!match_ident(".fo", ".foo.bar.Baz", Some("buzz"))); + assert!(!match_ident(".foo.", ".foo.bar.Baz", Some("buzz"))); + assert!(!match_ident(".buzz", ".foo.bar.Baz", Some("buzz"))); + assert!(!match_ident(".Baz.buzz", ".foo.bar.Baz", Some("buzz"))); + + // Suffix matches + assert!(match_ident("buzz", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident("Baz.buzz", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident("bar.Baz.buzz", ".foo.bar.Baz", Some("buzz"))); + assert!(match_ident( + "foo.bar.Baz.buzz", + ".foo.bar.Baz", + Some("buzz") + )); + + assert!(!match_ident("buz", ".foo.bar.Baz", Some("buzz"))); + assert!(!match_ident("uz", ".foo.bar.Baz", Some("buzz"))); + + // Type names + assert!(match_ident("Baz", ".foo.bar.Baz", None)); + assert!(match_ident(".", ".foo.bar.Baz", None)); + assert!(match_ident(".foo.bar", ".foo.bar.Baz", None)); + assert!(match_ident(".foo.bar.Baz", ".foo.bar.Baz", None)); + assert!(!match_ident(".fo", ".foo.bar.Baz", None)); + assert!(!match_ident(".buzz.Baz", ".foo.bar.Baz", None)); + } +} diff --git a/third_party/prost/prost-build/src/lib.rs b/third_party/prost/prost-build/src/lib.rs new file mode 100644 index 00000000000..72c66f5f4f3 --- /dev/null +++ b/third_party/prost/prost-build/src/lib.rs @@ -0,0 +1,813 @@ +#![doc(html_root_url = "https://docs.rs/prost-build/0.6.1")] +#![allow(clippy::option_as_ref_deref)] + +//! `prost-build` compiles `.proto` files into Rust. +//! +//! `prost-build` is designed to be used for build-time code generation as part of a Cargo +//! build-script. +//! +//! ## Example +//! +//! Let's create a small crate, `snazzy`, that defines a collection of +//! snazzy new items in a protobuf file. +//! +//! ```bash +//! $ cargo new snazzy && cd snazzy +//! ``` +//! +//! First, add `prost-build`, `prost` and its public dependencies to `Cargo.toml` +//! (see [crates.io](https://crates.io/crates/prost) for the current versions): +//! +//! ```toml +//! [dependencies] +//! bytes = +//! prost = +//! +//! [build-dependencies] +//! prost-build = { version = } +//! ``` +//! +//! Next, add `src/items.proto` to the project: +//! +//! ```proto +//! syntax = "proto3"; +//! +//! package snazzy.items; +//! +//! // A snazzy new shirt! +//! message Shirt { +//! enum Size { +//! SMALL = 0; +//! MEDIUM = 1; +//! LARGE = 2; +//! } +//! +//! string color = 1; +//! Size size = 2; +//! } +//! ``` +//! +//! To generate Rust code from `items.proto`, we use `prost-build` in the crate's +//! `build.rs` build-script: +//! +//! ```rust,no_run +//! # use std::io::Result; +//! fn main() -> Result<()> { +//! prost_build::compile_protos(&["src/items.proto"], &["src/"])?; +//! Ok(()) +//! } +//! ``` +//! +//! And finally, in `lib.rs`, include the generated code: +//! +//! ```rust,ignore +//! // Include the `items` module, which is generated from items.proto. +//! pub mod items { +//! include!(concat!(env!("OUT_DIR"), "/snazzy.items.rs")); +//! } +//! +//! pub fn create_large_shirt(color: String) -> items::Shirt { +//! let mut shirt = items::Shirt::default(); +//! shirt.color = color; +//! shirt.set_size(items::shirt::Size::Large); +//! shirt +//! } +//! ``` +//! +//! That's it! Run `cargo doc` to see documentation for the generated code. The full +//! example project can be found on [GitHub](https://github.com/danburkert/snazzy). +//! +//! ## Sourcing `protoc` +//! +//! `prost-build` depends on the Protocol Buffers compiler, `protoc`, to parse `.proto` files into +//! a representation that can be transformed into Rust. If set, `prost-build` uses the `PROTOC` and +//! `PROTOC_INCLUDE` environment variables for locating `protoc` and the Protobuf includes +//! directory. For example, on a macOS system where Protobuf is installed with Homebrew, set the +//! environment to: +//! +//! ```bash +//! PROTOC=/usr/local/bin/protoc +//! PROTOC_INCLUDE=/usr/local/include +//! ``` +//! +//! and in a typical Linux installation: +//! +//! ```bash +//! PROTOC=/usr/bin/protoc +//! PROTOC_INCLUDE=/usr/include +//! ``` +//! +//! If `PROTOC` is not found in the environment, then a pre-compiled `protoc` binary bundled in +//! the prost-build crate is used. Pre-compiled `protoc` binaries exist for Linux, macOS, and +//! Windows systems. If no pre-compiled `protoc` is available for the host platform, then the +//! `protoc` or `protoc.exe` binary on the `PATH` is used. If `protoc` is not available in any of +//! these fallback locations, then the build fails. +//! +//! If `PROTOC_INCLUDE` is not found in the environment, then the Protobuf include directory bundled +//! in the prost-build crate is be used. + +mod ast; +mod code_generator; +mod extern_paths; +mod ident; +mod message_graph; + +use std::collections::HashMap; +use std::default; +use std::env; +use std::fmt; +use std::fs; +use std::io::{Error, ErrorKind, Result}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use log::trace; +use prost::Message; +use prost_types::{FileDescriptorProto, FileDescriptorSet}; + +pub use crate::ast::{Comments, Method, Service}; +use crate::code_generator::CodeGenerator; +use crate::extern_paths::ExternPaths; +use crate::ident::to_snake; +use crate::message_graph::MessageGraph; + +type Module = Vec; + +/// A service generator takes a service descriptor and generates Rust code. +/// +/// `ServiceGenerator` can be used to generate application-specific interfaces +/// or implementations for Protobuf service definitions. +/// +/// Service generators are registered with a code generator using the +/// `Config::service_generator` method. +/// +/// A viable scenario is that an RPC framework provides a service generator. It generates a trait +/// describing methods of the service and some glue code to call the methods of the trait, defining +/// details like how errors are handled or if it is asynchronous. Then the user provides an +/// implementation of the generated trait in the application code and plugs it into the framework. +/// +/// Such framework isn't part of Prost at present. +pub trait ServiceGenerator { + /// Generates a Rust interface or implementation for a service, writing the + /// result to `buf`. + fn generate(&mut self, service: Service, buf: &mut String); + + /// Finalizes the generation process. + /// + /// In case there's something that needs to be output at the end of the generation process, it + /// goes here. Similar to [`generate`](#method.generate), the output should be appended to + /// `buf`. + /// + /// An example can be a module or other thing that needs to appear just once, not for each + /// service generated. + /// + /// This still can be called multiple times in a lifetime of the service generator, because it + /// is called once per `.proto` file. + /// + /// The default implementation is empty and does nothing. + fn finalize(&mut self, _buf: &mut String) {} + + /// Finalizes the generation process for an entire protobuf package. + /// + /// This differs from [`finalize`](#method.finalize) by where (and how often) it is called + /// during the service generator life cycle. This method is called once per protobuf package, + /// making it ideal for grouping services within a single package spread across multiple + /// `.proto` files. + /// + /// The default implementation is empty and does nothing. + fn finalize_package(&mut self, _package: &str, _buf: &mut String) {} +} + +/// Configuration options for Protobuf code generation. +/// +/// This configuration builder can be used to set non-default code generation options. +pub struct Config { + service_generator: Option>, + btree_map: Vec, + type_attributes: Vec<(String, String)>, + field_attributes: Vec<(String, String)>, + prost_types: bool, + strip_enum_prefix: bool, + out_dir: Option, + extern_paths: Vec<(String, String)>, +} + +impl Config { + /// Creates a new code generator configuration with default options. + pub fn new() -> Config { + Config::default() + } + + /// Configure the code generator to generate Rust [`BTreeMap`][1] fields for Protobuf + /// [`map`][2] type fields. + /// + /// # Arguments + /// + /// **`paths`** - paths to specific fields, messages, or packages which should use a Rust + /// `BTreeMap` for Protobuf `map` fields. Paths are specified in terms of the Protobuf type + /// name (not the generated Rust type name). Paths with a leading `.` are treated as fully + /// qualified names. Paths without a leading `.` are treated as relative, and are suffix + /// matched on the fully qualified field name. If a Protobuf map field matches any of the + /// paths, a Rust `BTreeMap` field is generated instead of the default [`HashMap`][3]. + /// + /// The matching is done on the Protobuf names, before converting to Rust-friendly casing + /// standards. + /// + /// # Examples + /// + /// ```rust + /// # let mut config = prost_build::Config::new(); + /// // Match a specific field in a message type. + /// config.btree_map(&[".my_messages.MyMessageType.my_map_field"]); + /// + /// // Match all map fields in a message type. + /// config.btree_map(&[".my_messages.MyMessageType"]); + /// + /// // Match all map fields in a package. + /// config.btree_map(&[".my_messages"]); + /// + /// // Match all map fields. Expecially useful in `no_std` contexts. + /// config.btree_map(&["."]); + /// + /// // Match all map fields in a nested message. + /// config.btree_map(&[".my_messages.MyMessageType.MyNestedMessageType"]); + /// + /// // Match all fields named 'my_map_field'. + /// config.btree_map(&["my_map_field"]); + /// + /// // Match all fields named 'my_map_field' in messages named 'MyMessageType', regardless of + /// // package or nesting. + /// config.btree_map(&["MyMessageType.my_map_field"]); + /// + /// // Match all fields named 'my_map_field', and all fields in the 'foo.bar' package. + /// config.btree_map(&["my_map_field", ".foo.bar"]); + /// ``` + /// + /// [1]: https://doc.rust-lang.org/std/collections/struct.BTreeMap.html + /// [2]: https://developers.google.com/protocol-buffers/docs/proto3#maps + /// [3]: https://doc.rust-lang.org/std/collections/struct.HashMap.html + pub fn btree_map(&mut self, paths: I) -> &mut Self + where + I: IntoIterator, + S: AsRef, + { + self.btree_map = paths.into_iter().map(|s| s.as_ref().to_string()).collect(); + self + } + + /// Add additional attribute to matched fields. + /// + /// # Arguments + /// + /// **`path`** - a patch matching any number of fields. These fields get the attribute. + /// For details about matching fields see [`btree_map`](#method.btree_map). + /// + /// **`attribute`** - an arbitrary string that'll be placed before each matched field. The + /// expected usage are additional attributes, usually in concert with whole-type + /// attributes set with [`type_attribute`](method.type_attribute), but it is not + /// checked and anything can be put there. + /// + /// Note that the calls to this method are cumulative ‒ if multiple paths from multiple calls + /// match the same field, the field gets all the corresponding attributes. + /// + /// # Examples + /// + /// ```rust + /// # let mut config = prost_build::Config::new(); + /// // Prost renames fields named `in` to `in_`. But if serialized through serde, + /// // they should as `in`. + /// config.field_attribute("in", "#[serde(rename = \"in\")]"); + /// ``` + pub fn field_attribute(&mut self, path: P, attribute: A) -> &mut Self + where + P: AsRef, + A: AsRef, + { + self.field_attributes + .push((path.as_ref().to_string(), attribute.as_ref().to_string())); + self + } + + /// Add additional attribute to matched messages, enums and one-ofs. + /// + /// # Arguments + /// + /// **`paths`** - a path matching any number of types. It works the same way as in + /// [`btree_map`](#method.btree_map), just with the field name omitted. + /// + /// **`attribute`** - an arbitrary string to be placed before each matched type. The + /// expected usage are additional attributes, but anything is allowed. + /// + /// The calls to this method are cumulative. They don't overwrite previous calls and if a + /// type is matched by multiple calls of the method, all relevant attributes are added to + /// it. + /// + /// For things like serde it might be needed to combine with [field + /// attributes](#method.field_attribute). + /// + /// # Examples + /// + /// ```rust + /// # let mut config = prost_build::Config::new(); + /// // Nothing around uses floats, so we can derive real `Eq` in addition to `PartialEq`. + /// config.type_attribute(".", "#[derive(Eq)]"); + /// // Some messages want to be serializable with serde as well. + /// config.type_attribute("my_messages.MyMessageType", + /// "#[derive(Serialize)] #[serde(rename-all = \"snake_case\")]"); + /// config.type_attribute("my_messages.MyMessageType.MyNestedMessageType", + /// "#[derive(Serialize)] #[serde(rename-all = \"snake_case\")]"); + /// ``` + /// + /// # Oneof fields + /// + /// The `oneof` fields don't have a type name of their own inside Protobuf. Therefore, the + /// field name can be used both with `type_attribute` and `field_attribute` ‒ the first is + /// placed before the `enum` type definition, the other before the field inside corresponding + /// message `struct`. + /// + /// In other words, to place an attribute on the `enum` implementing the `oneof`, the match + /// would look like `my_messages.MyMessageType.oneofname`. + pub fn type_attribute(&mut self, path: P, attribute: A) -> &mut Self + where + P: AsRef, + A: AsRef, + { + self.type_attributes + .push((path.as_ref().to_string(), attribute.as_ref().to_string())); + self + } + + /// Configures the code generator to use the provided service generator. + pub fn service_generator(&mut self, service_generator: Box) -> &mut Self { + self.service_generator = Some(service_generator); + self + } + + /// Configures the code generator to not use the `prost_types` crate for Protobuf well-known + /// types, and instead generate Protobuf well-known types from their `.proto` definitions. + pub fn compile_well_known_types(&mut self) -> &mut Self { + self.prost_types = false; + self + } + + /// Declare an externally provided Protobuf package or type. + /// + /// `extern_path` allows `prost` types in external crates to be referenced in generated code. + /// + /// When `prost` compiles a `.proto` which includes an import of another `.proto`, it will + /// automatically recursively compile the imported file as well. `extern_path` can be used + /// to instead substitute types from an external crate. + /// + /// # Example + /// + /// As an example, consider a crate, `uuid`, with a `prost`-generated `Uuid` type: + /// + /// ```proto + /// // uuid.proto + /// + /// syntax = "proto3"; + /// package uuid; + /// + /// message Uuid { + /// string uuid_str = 1; + /// } + /// ``` + /// + /// The `uuid` crate implements some traits for `Uuid`, and publicly exports it: + /// + /// ```rust,ignore + /// // lib.rs in the uuid crate + /// + /// include!(concat!(env!("OUT_DIR"), "/uuid.rs")); + /// + /// pub trait DoSomething { + /// fn do_it(&self); + /// } + /// + /// impl DoSomething for Uuid { + /// fn do_it(&self) { + /// println!("Done"); + /// } + /// } + /// ``` + /// + /// A separate crate, `my_application`, uses `prost` to generate message types which reference + /// `Uuid`: + /// + /// ```proto + /// // my_application.proto + /// + /// syntax = "proto3"; + /// package my_application; + /// + /// import "uuid.proto"; + /// + /// message MyMessage { + /// uuid.Uuid message_id = 1; + /// string some_payload = 2; + /// } + /// ``` + /// + /// Additionally, `my_application` depends on the trait impls provided by the `uuid` crate: + /// + /// ```rust,ignore + /// // `main.rs` of `my_application` + /// + /// use uuid::{DoSomething, Uuid}; + /// + /// include!(concat!(env!("OUT_DIR"), "/my_application.rs")); + /// + /// pub fn process_message(msg: MyMessage) { + /// if let Some(uuid) = msg.message_id { + /// uuid.do_it(); + /// } + /// } + /// ``` + /// + /// Without configuring `uuid` as an external path in `my_application`'s `build.rs`, `prost` + /// would compile a completely separate version of the `Uuid` type, and `process_message` would + /// fail to compile. However, if `my_application` configures `uuid` as an extern path with a + /// call to `.extern_path(".uuid", "::uuid")`, `prost` will use the external type instead of + /// compiling a new version of `Uuid`. Note that the configuration could also be specified as + /// `.extern_path(".uuid.Uuid", "::uuid::Uuid")` if only the `Uuid` type were externally + /// provided, and not the whole `uuid` package. + /// + /// # Usage + /// + /// `extern_path` takes a fully-qualified Protobuf path, and the corresponding Rust path that + /// it will be substituted with in generated code. The Protobuf path can refer to a package or + /// a type, and the Rust path should correspondingly refer to a Rust module or type. + /// + /// ```rust + /// # let mut config = prost_build::Config::new(); + /// // Declare the `uuid` Protobuf package and all nested packages and types as externally + /// // provided by the `uuid` crate. + /// config.extern_path(".uuid", "::uuid"); + /// + /// // Declare the `foo.bar.baz` Protobuf package and all nested packages and types as + /// // externally provided by the `foo_bar_baz` crate. + /// config.extern_path(".foo.bar.baz", "::foo_bar_baz"); + /// + /// // Declare the `uuid.Uuid` Protobuf type (and all nested types) as externally provided + /// // by the `uuid` crate's `Uuid` type. + /// config.extern_path(".uuid.Uuid", "::uuid::Uuid"); + /// ``` + pub fn extern_path(&mut self, proto_path: P1, rust_path: P2) -> &mut Self + where + P1: Into, + P2: Into, + { + self.extern_paths + .push((proto_path.into(), rust_path.into())); + self + } + + /// Configures the code generator to not strip the enum name from variant names. + /// + /// Protobuf enum definitions commonly include the enum name as a prefix of every variant name. + /// This style is non-idiomatic in Rust, so by default `prost` strips the enum name prefix from + /// variants which include it. Configuring this option prevents `prost` from stripping the + /// prefix. + pub fn retain_enum_prefix(&mut self) -> &mut Self { + self.strip_enum_prefix = false; + self + } + + /// Configures the output directory where generated Rust files will be written. + /// + /// If unset, defaults to the `OUT_DIR` environment variable. `OUT_DIR` is set by Cargo when + /// executing build scripts, so `out_dir` typically does not need to be configured. + pub fn out_dir

(&mut self, path: P) -> &mut Self + where + P: Into, + { + self.out_dir = Some(path.into()); + self + } + + /// Compile `.proto` files into Rust files during a Cargo build with additional code generator + /// configuration options. + /// + /// This method is like the `prost_build::compile_protos` function, with the added ability to + /// specify non-default code generation options. See that function for more information about + /// the arguments and generated outputs. + /// + /// # Example `build.rs` + /// + /// ```rust,no_run + /// # use std::io::Result; + /// fn main() -> Result<()> { + /// let mut prost_build = prost_build::Config::new(); + /// prost_build.btree_map(&["."]); + /// prost_build.compile_protos(&["src/frontend.proto", "src/backend.proto"], &["src"])?; + /// Ok(()) + /// } + /// ``` + pub fn compile_protos

(&mut self, protos: &[P], includes: &[P]) -> Result<()> + where + P: AsRef, + { + let target: PathBuf = self.out_dir.clone().map(Ok).unwrap_or_else(|| { + env::var_os("OUT_DIR") + .ok_or_else(|| { + Error::new(ErrorKind::Other, "OUT_DIR environment variable is not set") + }) + .map(Into::into) + })?; + + // TODO: This should probably emit 'rerun-if-changed=PATH' directives for cargo, however + // according to [1] if any are output then those paths replace the default crate root, + // which is undesirable. Figure out how to do it in an additive way; perhaps gcc-rs has + // this figured out. + // [1]: http://doc.crates.io/build-script.html#outputs-of-the-build-script + + let tmp = tempfile::Builder::new().prefix("prost-build").tempdir()?; + let descriptor_set = tmp.path().join("prost-descriptor-set"); + + let mut cmd = Command::new(protoc()); + cmd.arg("--include_imports") + .arg("--include_source_info") + .arg("-o") + .arg(&descriptor_set); + + for include in includes { + cmd.arg("-I").arg(include.as_ref()); + } + + // Set the protoc include after the user includes in case the user wants to + // override one of the built-in .protos. + cmd.arg("-I").arg(protoc_include()); + + for proto in protos { + cmd.arg(proto.as_ref()); + } + + let output = cmd.output()?; + if !output.status.success() { + return Err(Error::new( + ErrorKind::Other, + format!("protoc failed: {}", String::from_utf8_lossy(&output.stderr)), + )); + } + + let buf = fs::read(descriptor_set)?; + let descriptor_set = FileDescriptorSet::decode(&*buf).map_err(|error| { + Error::new( + ErrorKind::InvalidInput, + format!("invalid FileDescriptorSet: {}", error.to_string()), + ) + })?; + + let modules = self.generate(descriptor_set.file)?; + for (module, content) in modules { + let mut filename = module.join("."); + filename.push_str(".rs"); + + let output_path = target.join(&filename); + + let previous_content = fs::read(&output_path); + + if previous_content + .map(|previous_content| previous_content == content.as_bytes()) + .unwrap_or(false) + { + trace!("unchanged: {:?}", filename); + } else { + trace!("writing: {:?}", filename); + fs::write(output_path, content)?; + } + } + + Ok(()) + } + + fn generate(&mut self, files: Vec) -> Result> { + let mut modules = HashMap::new(); + let mut packages = HashMap::new(); + + let message_graph = MessageGraph::new(&files) + .map_err(|error| Error::new(ErrorKind::InvalidInput, error))?; + let extern_paths = ExternPaths::new(&self.extern_paths, self.prost_types) + .map_err(|error| Error::new(ErrorKind::InvalidInput, error))?; + + for file in files { + let module = self.module(&file); + + // Only record packages that have services + if !file.service.is_empty() { + packages.insert(module.clone(), file.package().to_string()); + } + + let mut buf = modules.entry(module).or_insert_with(String::new); + CodeGenerator::generate(self, &message_graph, &extern_paths, file, &mut buf); + } + + if let Some(ref mut service_generator) = self.service_generator { + for (module, package) in packages { + let buf = modules.get_mut(&module).unwrap(); + service_generator.finalize_package(&package, buf); + } + } + + Ok(modules) + } + + fn module(&self, file: &FileDescriptorProto) -> Module { + file.package() + .split('.') + .filter(|s| !s.is_empty()) + .map(to_snake) + .collect() + } +} + +impl default::Default for Config { + fn default() -> Config { + Config { + service_generator: None, + btree_map: Vec::new(), + type_attributes: Vec::new(), + field_attributes: Vec::new(), + prost_types: true, + strip_enum_prefix: true, + out_dir: None, + extern_paths: Vec::new(), + } + } +} + +impl fmt::Debug for Config { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Config") + .field("btree_map", &self.btree_map) + .field("type_attributes", &self.type_attributes) + .field("field_attributes", &self.field_attributes) + .field("prost_types", &self.prost_types) + .field("strip_enum_prefix", &self.strip_enum_prefix) + .field("out_dir", &self.out_dir) + .field("extern_paths", &self.extern_paths) + .finish() + } +} + +/// Compile `.proto` files into Rust files during a Cargo build. +/// +/// The generated `.rs` files are written to the Cargo `OUT_DIR` directory, suitable for use with +/// the [include!][1] macro. See the [Cargo `build.rs` code generation][2] example for more info. +/// +/// This function should be called in a project's `build.rs`. +/// +/// # Arguments +/// +/// **`protos`** - Paths to `.proto` files to compile. Any transitively [imported][3] `.proto` +/// files are automatically be included. +/// +/// **`includes`** - Paths to directories in which to search for imports. Directories are searched +/// in order. The `.proto` files passed in **`protos`** must be found in one of the provided +/// include directories. +/// +/// # Errors +/// +/// This function can fail for a number of reasons: +/// +/// - Failure to locate or download `protoc`. +/// - Failure to parse the `.proto`s. +/// - Failure to locate an imported `.proto`. +/// - Failure to compile a `.proto` without a [package specifier][4]. +/// +/// It's expected that this function call be `unwrap`ed in a `build.rs`; there is typically no +/// reason to gracefully recover from errors during a build. +/// +/// # Example `build.rs` +/// +/// ```rust,no_run +/// # use std::io::Result; +/// fn main() -> Result<()> { +/// prost_build::compile_protos(&["src/frontend.proto", "src/backend.proto"], &["src"])?; +/// Ok(()) +/// } +/// ``` +/// +/// [1]: https://doc.rust-lang.org/std/macro.include.html +/// [2]: http://doc.crates.io/build-script.html#case-study-code-generation +/// [3]: https://developers.google.com/protocol-buffers/docs/proto3#importing-definitions +/// [4]: https://developers.google.com/protocol-buffers/docs/proto#packages +pub fn compile_protos

(protos: &[P], includes: &[P]) -> Result<()> +where + P: AsRef, +{ + Config::new().compile_protos(protos, includes) +} + +/// Returns the path to the `protoc` binary. +pub fn protoc() -> PathBuf { + match env::var_os("PROTOC") { + Some(protoc) => PathBuf::from(protoc), + None => PathBuf::from(env!("PROTOC")), + } +} + +/// Returns the path to the Protobuf include directory. +pub fn protoc_include() -> PathBuf { + match env::var_os("PROTOC_INCLUDE") { + Some(include) => PathBuf::from(include), + None => PathBuf::from(env!("PROTOC_INCLUDE")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::rc::Rc; + + /// An example service generator that generates a trait with methods corresponding to the + /// service methods. + struct ServiceTraitGenerator; + impl ServiceGenerator for ServiceTraitGenerator { + fn generate(&mut self, service: Service, buf: &mut String) { + // Generate a trait for the service. + service.comments.append_with_indent(0, buf); + buf.push_str(&format!("trait {} {{\n", &service.name)); + + // Generate the service methods. + for method in service.methods { + method.comments.append_with_indent(1, buf); + buf.push_str(&format!( + " fn {}({}) -> {};\n", + method.name, method.input_type, method.output_type + )); + } + + // Close out the trait. + buf.push_str("}\n"); + } + fn finalize(&mut self, buf: &mut String) { + // Needs to be present only once, no matter how many services there are + buf.push_str("pub mod utils { }\n"); + } + } + + /// Implements `ServiceGenerator` and provides some state for assertions. + struct MockServiceGenerator { + state: Rc>, + } + + /// Holds state for `MockServiceGenerator` + #[derive(Default)] + struct MockState { + service_names: Vec, + package_names: Vec, + finalized: u32, + } + + impl MockServiceGenerator { + fn new(state: Rc>) -> Self { + Self { state } + } + } + + impl ServiceGenerator for MockServiceGenerator { + fn generate(&mut self, service: Service, _buf: &mut String) { + let mut state = self.state.borrow_mut(); + state.service_names.push(service.name); + } + + fn finalize(&mut self, _buf: &mut String) { + let mut state = self.state.borrow_mut(); + state.finalized += 1; + } + + fn finalize_package(&mut self, package: &str, _buf: &mut String) { + let mut state = self.state.borrow_mut(); + state.package_names.push(package.to_string()); + } + } + + #[test] + fn smoke_test() { + let _ = env_logger::try_init(); + Config::new() + .service_generator(Box::new(ServiceTraitGenerator)) + .compile_protos(&["src/smoke_test.proto"], &["src"]) + .unwrap(); + } + + #[test] + fn finalize_package() { + let _ = env_logger::try_init(); + + let state = Rc::new(RefCell::new(MockState::default())); + let gen = MockServiceGenerator::new(Rc::clone(&state)); + + Config::new() + .service_generator(Box::new(gen)) + .compile_protos(&["src/hello.proto", "src/goodbye.proto"], &["src"]) + .unwrap(); + + let state = state.borrow(); + assert_eq!(&state.service_names, &["Greeting", "Farewell"]); + assert_eq!(&state.package_names, &["helloworld"]); + assert_eq!(state.finalized, 3); + } +} diff --git a/third_party/prost/prost-build/src/message_graph.rs b/third_party/prost/prost-build/src/message_graph.rs new file mode 100644 index 00000000000..e66066c2b19 --- /dev/null +++ b/third_party/prost/prost-build/src/message_graph.rs @@ -0,0 +1,91 @@ +use std::collections::HashMap; + +use petgraph::algo::has_path_connecting; +use petgraph::graph::NodeIndex; +use petgraph::Graph; + +use prost_types::{field_descriptor_proto, DescriptorProto, FileDescriptorProto}; + +/// `MessageGraph` builds a graph of messages whose edges correspond to nesting. +/// The goal is to recognize when message types are recursively nested, so +/// that fields can be boxed when necessary. +pub struct MessageGraph { + index: HashMap, + graph: Graph, +} + +impl MessageGraph { + pub fn new(files: &[FileDescriptorProto]) -> Result { + let mut msg_graph = MessageGraph { + index: HashMap::new(), + graph: Graph::new(), + }; + + for file in files { + let package = format!( + ".{}", + file.package.as_ref().ok_or_else(|| { + format!( + "prost requires a package specifier in all .proto files \ + (https://developers.google.com/protocol-buffers/docs/proto#packages); \ + file with missing package specifier: {}", + file.name.as_ref().map_or("(unknown)", String::as_ref), + ) + })?, + ); + for msg in &file.message_type { + msg_graph.add_message(&package, msg); + } + } + + Ok(msg_graph) + } + + fn get_or_insert_index(&mut self, msg_name: String) -> NodeIndex { + let MessageGraph { + ref mut index, + ref mut graph, + } = *self; + assert_eq!(b'.', msg_name.as_bytes()[0]); + *index + .entry(msg_name.clone()) + .or_insert_with(|| graph.add_node(msg_name)) + } + + /// Adds message to graph IFF it contains a non-repeated field containing another message. + /// The purpose of the message graph is detecting recursively nested messages and co-recursively nested messages. + /// Because prost does not box message fields, recursively nested messages would not compile in Rust. + /// To allow recursive messages, the message graph is used to detect recursion and automatically box the recursive field. + /// Since repeated messages are already put in a Vec, boxing them isn’t necessary even if the reference is recursive. + fn add_message(&mut self, package: &str, msg: &DescriptorProto) { + let msg_name = format!("{}.{}", package, msg.name.as_ref().unwrap()); + let msg_index = self.get_or_insert_index(msg_name.clone()); + + for field in &msg.field { + if field.r#type() == field_descriptor_proto::Type::Message + && field.label() != field_descriptor_proto::Label::Repeated + { + let field_index = self.get_or_insert_index(field.type_name.clone().unwrap()); + self.graph.add_edge(msg_index, field_index, ()); + } + } + + for msg in &msg.nested_type { + self.add_message(&msg_name, msg); + } + } + + /// Returns true if message type `inner` is nested in message type `outer`. + pub fn is_nested(&self, outer: &str, inner: &str) -> bool { + let outer = match self.index.get(outer) { + Some(outer) => *outer, + None => return false, + }; + let inner = match self.index.get(inner) { + Some(inner) => *inner, + None => return false, + }; + + has_path_connecting(&self.graph, outer, inner, None) + } +} diff --git a/third_party/prost/prost-build/src/smoke_test.proto b/third_party/prost/prost-build/src/smoke_test.proto new file mode 100644 index 00000000000..04679a27710 --- /dev/null +++ b/third_party/prost/prost-build/src/smoke_test.proto @@ -0,0 +1,18 @@ +syntax = "proto2"; + +package smoke_test; + +message SmokeRequest { +} + +message SmokeResponse { +} + +// Just a smoke test service. +service SmokeService { + + // A detached comment block. + + // Blow some smoke. + rpc BlowSmoke(SmokeRequest) returns (SmokeResponse); +} diff --git a/third_party/prost/prost-build/src/types.proto b/third_party/prost/prost-build/src/types.proto new file mode 100644 index 00000000000..4d9d5e0e2c8 --- /dev/null +++ b/third_party/prost/prost-build/src/types.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package helloworld; + +message Message { + string say = 1; +} + +message Response { + string say = 1; +} diff --git a/third_party/prost/prost-derive/Cargo.toml b/third_party/prost/prost-derive/Cargo.toml new file mode 100644 index 00000000000..1fec0726b70 --- /dev/null +++ b/third_party/prost/prost-derive/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "prost-derive" +version = "0.6.1" +authors = ["Dan Burkert "] +license = "Apache-2.0" +repository = "https://github.com/danburkert/prost" +documentation = "https://docs.rs/prost-derive" +readme = "README.md" +description = "A Protocol Buffers implementation for the Rust Language." +edition = "2018" + +[lib] +proc_macro = true + +[dependencies] +anyhow = "1" +itertools = "0.9" +proc-macro2 = "1" +quote = "1" +syn = { version = "1", features = [ "extra-traits" ] } diff --git a/third_party/prost/prost-derive/README.md b/third_party/prost/prost-derive/README.md new file mode 100644 index 00000000000..a51050e7e66 --- /dev/null +++ b/third_party/prost/prost-derive/README.md @@ -0,0 +1,16 @@ +[![Documentation](https://docs.rs/prost-derive/badge.svg)](https://docs.rs/prost-derive/) +[![Crate](https://img.shields.io/crates/v/prost-derive.svg)](https://crates.io/crates/prost-derive) + +# prost-derive + +`prost-derive` handles generating encoding and decoding implementations for Rust +types annotated with `prost` annotation. For the most part, users of `prost` +shouldn't need to interact with `prost-derive` directly. + +## License + +`prost-derive` is distributed under the terms of the Apache License (Version 2.0). + +See [LICENSE](../LICENSE) for details. + +Copyright 2017 Dan Burkert diff --git a/third_party/prost/prost-derive/src/field/group.rs b/third_party/prost/prost-derive/src/field/group.rs new file mode 100644 index 00000000000..6396128802d --- /dev/null +++ b/third_party/prost/prost-derive/src/field/group.rs @@ -0,0 +1,134 @@ +use anyhow::{bail, Error}; +use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; +use syn::Meta; + +use crate::field::{set_bool, set_option, tag_attr, word_attr, Label}; + +#[derive(Clone)] +pub struct Field { + pub label: Label, + pub tag: u32, +} + +impl Field { + pub fn new(attrs: &[Meta], inferred_tag: Option) -> Result, Error> { + let mut group = false; + let mut label = None; + let mut tag = None; + let mut boxed = false; + + let mut unknown_attrs = Vec::new(); + + for attr in attrs { + if word_attr("group", attr) { + set_bool(&mut group, "duplicate group attributes")?; + } else if word_attr("boxed", attr) { + set_bool(&mut boxed, "duplicate boxed attributes")?; + } else if let Some(t) = tag_attr(attr)? { + set_option(&mut tag, t, "duplicate tag attributes")?; + } else if let Some(l) = Label::from_attr(attr) { + set_option(&mut label, l, "duplicate label attributes")?; + } else { + unknown_attrs.push(attr); + } + } + + if !group { + return Ok(None); + } + + match unknown_attrs.len() { + 0 => (), + 1 => bail!("unknown attribute for group field: {:?}", unknown_attrs[0]), + _ => bail!("unknown attributes for group field: {:?}", unknown_attrs), + } + + let tag = match tag.or(inferred_tag) { + Some(tag) => tag, + None => bail!("group field is missing a tag attribute"), + }; + + Ok(Some(Field { + label: label.unwrap_or(Label::Optional), + tag, + })) + } + + pub fn new_oneof(attrs: &[Meta]) -> Result, Error> { + if let Some(mut field) = Field::new(attrs, None)? { + if let Some(attr) = attrs.iter().find(|attr| Label::from_attr(attr).is_some()) { + bail!( + "invalid attribute for oneof field: {}", + attr.path().into_token_stream() + ); + } + field.label = Label::Required; + Ok(Some(field)) + } else { + Ok(None) + } + } + + pub fn encode(&self, ident: TokenStream) -> TokenStream { + let tag = self.tag; + match self.label { + Label::Optional => quote! { + if let Some(ref msg) = #ident { + ::prost::encoding::group::encode(#tag, msg, buf); + } + }, + Label::Required => quote! { + ::prost::encoding::group::encode(#tag, &#ident, buf); + }, + Label::Repeated => quote! { + for msg in &#ident { + ::prost::encoding::group::encode(#tag, msg, buf); + } + }, + } + } + + pub fn merge(&self, ident: TokenStream) -> TokenStream { + match self.label { + Label::Optional => quote! { + ::prost::encoding::group::merge( + tag, + wire_type, + #ident.get_or_insert_with(Default::default), + buf, + ctx, + ) + }, + Label::Required => quote! { + ::prost::encoding::group::merge(tag, wire_type, #ident, buf, ctx) + }, + Label::Repeated => quote! { + ::prost::encoding::group::merge_repeated(tag, wire_type, #ident, buf, ctx) + }, + } + } + + pub fn encoded_len(&self, ident: TokenStream) -> TokenStream { + let tag = self.tag; + match self.label { + Label::Optional => quote! { + #ident.as_ref().map_or(0, |msg| ::prost::encoding::group::encoded_len(#tag, msg)) + }, + Label::Required => quote! { + ::prost::encoding::group::encoded_len(#tag, &#ident) + }, + Label::Repeated => quote! { + ::prost::encoding::group::encoded_len_repeated(#tag, &#ident) + }, + } + } + + pub fn clear(&self, ident: TokenStream) -> TokenStream { + match self.label { + Label::Optional => quote!(#ident = ::core::option::Option::None), + Label::Required => quote!(#ident.clear()), + Label::Repeated => quote!(#ident.clear()), + } + } +} diff --git a/third_party/prost/prost-derive/src/field/map.rs b/third_party/prost/prost-derive/src/field/map.rs new file mode 100644 index 00000000000..1228a6fae29 --- /dev/null +++ b/third_party/prost/prost-derive/src/field/map.rs @@ -0,0 +1,394 @@ +use anyhow::{bail, Error}; +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::{Ident, Lit, Meta, MetaNameValue, NestedMeta}; + +use crate::field::{scalar, set_option, tag_attr}; + +#[derive(Clone, Debug)] +pub enum MapTy { + HashMap, + BTreeMap, +} + +impl MapTy { + fn from_str(s: &str) -> Option { + match s { + "map" | "hash_map" => Some(MapTy::HashMap), + "btree_map" => Some(MapTy::BTreeMap), + _ => None, + } + } + + fn module(&self) -> Ident { + match *self { + MapTy::HashMap => Ident::new("hash_map", Span::call_site()), + MapTy::BTreeMap => Ident::new("btree_map", Span::call_site()), + } + } + + fn lib(&self) -> TokenStream { + match self { + MapTy::HashMap => quote! { std }, + MapTy::BTreeMap => quote! { prost::alloc }, + } + } +} + +fn fake_scalar(ty: scalar::Ty) -> scalar::Field { + let kind = scalar::Kind::Plain(scalar::DefaultValue::new(&ty)); + scalar::Field { + ty, + kind, + tag: 0, // Not used here + } +} + +#[derive(Clone)] +pub struct Field { + pub map_ty: MapTy, + pub key_ty: scalar::Ty, + pub value_ty: ValueTy, + pub tag: u32, +} + +impl Field { + pub fn new(attrs: &[Meta], inferred_tag: Option) -> Result, Error> { + let mut types = None; + let mut tag = None; + + for attr in attrs { + if let Some(t) = tag_attr(attr)? { + set_option(&mut tag, t, "duplicate tag attributes")?; + } else if let Some(map_ty) = attr + .path() + .get_ident() + .and_then(|i| MapTy::from_str(&i.to_string())) + { + let (k, v): (String, String) = match &*attr { + Meta::NameValue(MetaNameValue { + lit: Lit::Str(lit), .. + }) => { + let items = lit.value(); + let mut items = items.split(',').map(ToString::to_string); + let k = items.next().unwrap(); + let v = match items.next() { + Some(k) => k, + None => bail!("invalid map attribute: must have key and value types"), + }; + if items.next().is_some() { + bail!("invalid map attribute: {:?}", attr); + } + (k, v) + } + Meta::List(meta_list) => { + // TODO(rustlang/rust#23121): slice pattern matching would make this much nicer. + if meta_list.nested.len() != 2 { + bail!("invalid map attribute: must contain key and value types"); + } + let k = match &meta_list.nested[0] { + NestedMeta::Meta(Meta::Path(k)) if k.get_ident().is_some() => { + k.get_ident().unwrap().to_string() + } + _ => bail!("invalid map attribute: key must be an identifier"), + }; + let v = match &meta_list.nested[1] { + NestedMeta::Meta(Meta::Path(v)) if v.get_ident().is_some() => { + v.get_ident().unwrap().to_string() + } + _ => bail!("invalid map attribute: value must be an identifier"), + }; + (k, v) + } + _ => return Ok(None), + }; + set_option( + &mut types, + (map_ty, key_ty_from_str(&k)?, ValueTy::from_str(&v)?), + "duplicate map type attribute", + )?; + } else { + return Ok(None); + } + } + + Ok(match (types, tag.or(inferred_tag)) { + (Some((map_ty, key_ty, value_ty)), Some(tag)) => Some(Field { + map_ty, + key_ty, + value_ty, + tag, + }), + _ => None, + }) + } + + pub fn new_oneof(attrs: &[Meta]) -> Result, Error> { + Field::new(attrs, None) + } + + /// Returns a statement which encodes the map field. + pub fn encode(&self, ident: TokenStream) -> TokenStream { + let tag = self.tag; + let key_mod = self.key_ty.module(); + let ke = quote!(::prost::encoding::#key_mod::encode); + let kl = quote!(::prost::encoding::#key_mod::encoded_len); + let module = self.map_ty.module(); + match &self.value_ty { + ValueTy::Scalar(scalar::Ty::Enumeration(ty)) => { + let default = quote!(#ty::default() as i32); + quote! { + ::prost::encoding::#module::encode_with_default( + #ke, + #kl, + ::prost::encoding::int32::encode, + ::prost::encoding::int32::encoded_len, + &(#default), + #tag, + &#ident, + buf, + ); + } + } + ValueTy::Scalar(value_ty) => { + let val_mod = value_ty.module(); + let ve = quote!(::prost::encoding::#val_mod::encode); + let vl = quote!(::prost::encoding::#val_mod::encoded_len); + quote! { + ::prost::encoding::#module::encode( + #ke, + #kl, + #ve, + #vl, + #tag, + &#ident, + buf, + ); + } + } + ValueTy::Message => quote! { + ::prost::encoding::#module::encode( + #ke, + #kl, + ::prost::encoding::message::encode, + ::prost::encoding::message::encoded_len, + #tag, + &#ident, + buf, + ); + }, + } + } + + /// Returns an expression which evaluates to the result of merging a decoded key value pair + /// into the map. + pub fn merge(&self, ident: TokenStream) -> TokenStream { + let key_mod = self.key_ty.module(); + let km = quote!(::prost::encoding::#key_mod::merge); + let module = self.map_ty.module(); + match &self.value_ty { + ValueTy::Scalar(scalar::Ty::Enumeration(ty)) => { + let default = quote!(#ty::default() as i32); + quote! { + ::prost::encoding::#module::merge_with_default( + #km, + ::prost::encoding::int32::merge, + #default, + &mut #ident, + buf, + ctx, + ) + } + } + ValueTy::Scalar(value_ty) => { + let val_mod = value_ty.module(); + let vm = quote!(::prost::encoding::#val_mod::merge); + quote!(::prost::encoding::#module::merge(#km, #vm, &mut #ident, buf, ctx)) + } + ValueTy::Message => quote! { + ::prost::encoding::#module::merge( + #km, + ::prost::encoding::message::merge, + &mut #ident, + buf, + ctx, + ) + }, + } + } + + /// Returns an expression which evaluates to the encoded length of the map. + pub fn encoded_len(&self, ident: TokenStream) -> TokenStream { + let tag = self.tag; + let key_mod = self.key_ty.module(); + let kl = quote!(::prost::encoding::#key_mod::encoded_len); + let module = self.map_ty.module(); + match &self.value_ty { + ValueTy::Scalar(scalar::Ty::Enumeration(ty)) => { + let default = quote!(#ty::default() as i32); + quote! { + ::prost::encoding::#module::encoded_len_with_default( + #kl, + ::prost::encoding::int32::encoded_len, + &(#default), + #tag, + &#ident, + ) + } + } + ValueTy::Scalar(value_ty) => { + let val_mod = value_ty.module(); + let vl = quote!(::prost::encoding::#val_mod::encoded_len); + quote!(::prost::encoding::#module::encoded_len(#kl, #vl, #tag, &#ident)) + } + ValueTy::Message => quote! { + ::prost::encoding::#module::encoded_len( + #kl, + ::prost::encoding::message::encoded_len, + #tag, + &#ident, + ) + }, + } + } + + pub fn clear(&self, ident: TokenStream) -> TokenStream { + quote!(#ident.clear()) + } + + /// Returns methods to embed in the message. + pub fn methods(&self, ident: &Ident) -> Option { + if let ValueTy::Scalar(scalar::Ty::Enumeration(ty)) = &self.value_ty { + let key_ty = self.key_ty.rust_type(); + let key_ref_ty = self.key_ty.rust_ref_type(); + + let get = Ident::new(&format!("get_{}", ident), Span::call_site()); + let insert = Ident::new(&format!("insert_{}", ident), Span::call_site()); + let take_ref = if self.key_ty.is_numeric() { + quote!(&) + } else { + quote!() + }; + + let get_doc = format!( + "Returns the enum value for the corresponding key in `{}`, \ + or `None` if the entry does not exist or it is not a valid enum value.", + ident, + ); + let insert_doc = format!("Inserts a key value pair into `{}`.", ident); + Some(quote! { + #[doc=#get_doc] + pub fn #get(&self, key: #key_ref_ty) -> ::core::option::Option<#ty> { + self.#ident.get(#take_ref key).cloned().and_then(#ty::from_i32) + } + #[doc=#insert_doc] + pub fn #insert(&mut self, key: #key_ty, value: #ty) -> ::core::option::Option<#ty> { + self.#ident.insert(key, value as i32).and_then(#ty::from_i32) + } + }) + } else { + None + } + } + + /// Returns a newtype wrapper around the map, implementing nicer Debug + /// + /// The Debug tries to convert any enumerations met into the variants if possible, instead of + /// outputting the raw numbers. + pub fn debug(&self, wrapper_name: TokenStream) -> TokenStream { + let type_name = match self.map_ty { + MapTy::HashMap => Ident::new("HashMap", Span::call_site()), + MapTy::BTreeMap => Ident::new("BTreeMap", Span::call_site()), + }; + + // A fake field for generating the debug wrapper + let key_wrapper = fake_scalar(self.key_ty.clone()).debug(quote!(KeyWrapper)); + let key = self.key_ty.rust_type(); + let value_wrapper = self.value_ty.debug(); + let libname = self.map_ty.lib(); + let fmt = quote! { + fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + #key_wrapper + #value_wrapper + let mut builder = f.debug_map(); + for (k, v) in self.0 { + builder.entry(&KeyWrapper(k), &ValueWrapper(v)); + } + builder.finish() + } + }; + match &self.value_ty { + ValueTy::Scalar(ty) => { + let value = ty.rust_type(); + quote! { + struct #wrapper_name<'a>(&'a ::#libname::collections::#type_name<#key, #value>); + impl<'a> ::core::fmt::Debug for #wrapper_name<'a> { + #fmt + } + } + } + ValueTy::Message => quote! { + struct #wrapper_name<'a, V: 'a>(&'a ::#libname::collections::#type_name<#key, V>); + impl<'a, V> ::core::fmt::Debug for #wrapper_name<'a, V> + where + V: ::core::fmt::Debug + 'a, + { + #fmt + } + }, + } + } +} + +fn key_ty_from_str(s: &str) -> Result { + let ty = scalar::Ty::from_str(s)?; + match ty { + scalar::Ty::Int32 + | scalar::Ty::Int64 + | scalar::Ty::Uint32 + | scalar::Ty::Uint64 + | scalar::Ty::Sint32 + | scalar::Ty::Sint64 + | scalar::Ty::Fixed32 + | scalar::Ty::Fixed64 + | scalar::Ty::Sfixed32 + | scalar::Ty::Sfixed64 + | scalar::Ty::Bool + | scalar::Ty::String => Ok(ty), + _ => bail!("invalid map key type: {}", s), + } +} + +/// A map value type. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ValueTy { + Scalar(scalar::Ty), + Message, +} + +impl ValueTy { + fn from_str(s: &str) -> Result { + if let Ok(ty) = scalar::Ty::from_str(s) { + Ok(ValueTy::Scalar(ty)) + } else if s.trim() == "message" { + Ok(ValueTy::Message) + } else { + bail!("invalid map value type: {}", s); + } + } + + /// Returns a newtype wrapper around the ValueTy for nicer debug. + /// + /// If the contained value is enumeration, it tries to convert it to the variant. If not, it + /// just forwards the implementation. + fn debug(&self) -> TokenStream { + match self { + ValueTy::Scalar(ty) => fake_scalar(ty.clone()).debug(quote!(ValueWrapper)), + ValueTy::Message => quote!( + fn ValueWrapper(v: T) -> T { + v + } + ), + } + } +} diff --git a/third_party/prost/prost-derive/src/field/message.rs b/third_party/prost/prost-derive/src/field/message.rs new file mode 100644 index 00000000000..1ff7c379839 --- /dev/null +++ b/third_party/prost/prost-derive/src/field/message.rs @@ -0,0 +1,134 @@ +use anyhow::{bail, Error}; +use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; +use syn::Meta; + +use crate::field::{set_bool, set_option, tag_attr, word_attr, Label}; + +#[derive(Clone)] +pub struct Field { + pub label: Label, + pub tag: u32, +} + +impl Field { + pub fn new(attrs: &[Meta], inferred_tag: Option) -> Result, Error> { + let mut message = false; + let mut label = None; + let mut tag = None; + let mut boxed = false; + + let mut unknown_attrs = Vec::new(); + + for attr in attrs { + if word_attr("message", attr) { + set_bool(&mut message, "duplicate message attribute")?; + } else if word_attr("boxed", attr) { + set_bool(&mut boxed, "duplicate boxed attribute")?; + } else if let Some(t) = tag_attr(attr)? { + set_option(&mut tag, t, "duplicate tag attributes")?; + } else if let Some(l) = Label::from_attr(attr) { + set_option(&mut label, l, "duplicate label attributes")?; + } else { + unknown_attrs.push(attr); + } + } + + if !message { + return Ok(None); + } + + match unknown_attrs.len() { + 0 => (), + 1 => bail!( + "unknown attribute for message field: {:?}", + unknown_attrs[0] + ), + _ => bail!("unknown attributes for message field: {:?}", unknown_attrs), + } + + let tag = match tag.or(inferred_tag) { + Some(tag) => tag, + None => bail!("message field is missing a tag attribute"), + }; + + Ok(Some(Field { + label: label.unwrap_or(Label::Optional), + tag, + })) + } + + pub fn new_oneof(attrs: &[Meta]) -> Result, Error> { + if let Some(mut field) = Field::new(attrs, None)? { + if let Some(attr) = attrs.iter().find(|attr| Label::from_attr(attr).is_some()) { + bail!( + "invalid attribute for oneof field: {}", + attr.path().into_token_stream() + ); + } + field.label = Label::Required; + Ok(Some(field)) + } else { + Ok(None) + } + } + + pub fn encode(&self, ident: TokenStream) -> TokenStream { + let tag = self.tag; + match self.label { + Label::Optional => quote! { + if let Some(ref msg) = #ident { + ::prost::encoding::message::encode(#tag, msg, buf); + } + }, + Label::Required => quote! { + ::prost::encoding::message::encode(#tag, &#ident, buf); + }, + Label::Repeated => quote! { + for msg in &#ident { + ::prost::encoding::message::encode(#tag, msg, buf); + } + }, + } + } + + pub fn merge(&self, ident: TokenStream) -> TokenStream { + match self.label { + Label::Optional => quote! { + ::prost::encoding::message::merge(wire_type, + #ident.get_or_insert_with(Default::default), + buf, + ctx) + }, + Label::Required => quote! { + ::prost::encoding::message::merge(wire_type, #ident, buf, ctx) + }, + Label::Repeated => quote! { + ::prost::encoding::message::merge_repeated(wire_type, #ident, buf, ctx) + }, + } + } + + pub fn encoded_len(&self, ident: TokenStream) -> TokenStream { + let tag = self.tag; + match self.label { + Label::Optional => quote! { + #ident.as_ref().map_or(0, |msg| ::prost::encoding::message::encoded_len(#tag, msg)) + }, + Label::Required => quote! { + ::prost::encoding::message::encoded_len(#tag, &#ident) + }, + Label::Repeated => quote! { + ::prost::encoding::message::encoded_len_repeated(#tag, &#ident) + }, + } + } + + pub fn clear(&self, ident: TokenStream) -> TokenStream { + match self.label { + Label::Optional => quote!(#ident = ::core::option::Option::None), + Label::Required => quote!(#ident.clear()), + Label::Repeated => quote!(#ident.clear()), + } + } +} diff --git a/third_party/prost/prost-derive/src/field/mod.rs b/third_party/prost/prost-derive/src/field/mod.rs new file mode 100644 index 00000000000..9e2e70c4c7a --- /dev/null +++ b/third_party/prost/prost-derive/src/field/mod.rs @@ -0,0 +1,366 @@ +mod group; +mod map; +mod message; +mod oneof; +mod scalar; + +use std::fmt; +use std::slice; + +use anyhow::{bail, Error}; +use proc_macro2::TokenStream; +use quote::quote; +use syn::{Attribute, Ident, Lit, LitBool, Meta, MetaList, MetaNameValue, NestedMeta}; + +#[derive(Clone)] +pub enum Field { + /// A scalar field. + Scalar(scalar::Field), + /// A message field. + Message(message::Field), + /// A map field. + Map(map::Field), + /// A oneof field. + Oneof(oneof::Field), + /// A group field. + Group(group::Field), +} + +impl Field { + /// Creates a new `Field` from an iterator of field attributes. + /// + /// If the meta items are invalid, an error will be returned. + /// If the field should be ignored, `None` is returned. + pub fn new(attrs: Vec, inferred_tag: Option) -> Result, Error> { + let attrs = prost_attrs(attrs)?; + + // TODO: check for ignore attribute. + + let field = if let Some(field) = scalar::Field::new(&attrs, inferred_tag)? { + Field::Scalar(field) + } else if let Some(field) = message::Field::new(&attrs, inferred_tag)? { + Field::Message(field) + } else if let Some(field) = map::Field::new(&attrs, inferred_tag)? { + Field::Map(field) + } else if let Some(field) = oneof::Field::new(&attrs)? { + Field::Oneof(field) + } else if let Some(field) = group::Field::new(&attrs, inferred_tag)? { + Field::Group(field) + } else { + bail!("no type attribute"); + }; + + Ok(Some(field)) + } + + /// Creates a new oneof `Field` from an iterator of field attributes. + /// + /// If the meta items are invalid, an error will be returned. + /// If the field should be ignored, `None` is returned. + pub fn new_oneof(attrs: Vec) -> Result, Error> { + let attrs = prost_attrs(attrs)?; + + // TODO: check for ignore attribute. + + let field = if let Some(field) = scalar::Field::new_oneof(&attrs)? { + Field::Scalar(field) + } else if let Some(field) = message::Field::new_oneof(&attrs)? { + Field::Message(field) + } else if let Some(field) = map::Field::new_oneof(&attrs)? { + Field::Map(field) + } else if let Some(field) = group::Field::new_oneof(&attrs)? { + Field::Group(field) + } else { + bail!("no type attribute for oneof field"); + }; + + Ok(Some(field)) + } + + pub fn tags(&self) -> Vec { + match *self { + Field::Scalar(ref scalar) => vec![scalar.tag], + Field::Message(ref message) => vec![message.tag], + Field::Map(ref map) => vec![map.tag], + Field::Oneof(ref oneof) => oneof.tags.clone(), + Field::Group(ref group) => vec![group.tag], + } + } + + /// Returns a statement which encodes the field. + pub fn encode(&self, ident: TokenStream) -> TokenStream { + match *self { + Field::Scalar(ref scalar) => scalar.encode(ident), + Field::Message(ref message) => message.encode(ident), + Field::Map(ref map) => map.encode(ident), + Field::Oneof(ref oneof) => oneof.encode(ident), + Field::Group(ref group) => group.encode(ident), + } + } + + /// Returns an expression which evaluates to the result of merging a decoded + /// value into the field. + pub fn merge(&self, ident: TokenStream) -> TokenStream { + match *self { + Field::Scalar(ref scalar) => scalar.merge(ident), + Field::Message(ref message) => message.merge(ident), + Field::Map(ref map) => map.merge(ident), + Field::Oneof(ref oneof) => oneof.merge(ident), + Field::Group(ref group) => group.merge(ident), + } + } + + /// Returns an expression which evaluates to the encoded length of the field. + pub fn encoded_len(&self, ident: TokenStream) -> TokenStream { + match *self { + Field::Scalar(ref scalar) => scalar.encoded_len(ident), + Field::Map(ref map) => map.encoded_len(ident), + Field::Message(ref msg) => msg.encoded_len(ident), + Field::Oneof(ref oneof) => oneof.encoded_len(ident), + Field::Group(ref group) => group.encoded_len(ident), + } + } + + /// Returns a statement which clears the field. + pub fn clear(&self, ident: TokenStream) -> TokenStream { + match *self { + Field::Scalar(ref scalar) => scalar.clear(ident), + Field::Message(ref message) => message.clear(ident), + Field::Map(ref map) => map.clear(ident), + Field::Oneof(ref oneof) => oneof.clear(ident), + Field::Group(ref group) => group.clear(ident), + } + } + + pub fn default(&self) -> TokenStream { + match *self { + Field::Scalar(ref scalar) => scalar.default(), + _ => quote!(::core::default::Default::default()), + } + } + + /// Produces the fragment implementing debug for the given field. + pub fn debug(&self, ident: TokenStream) -> TokenStream { + match *self { + Field::Scalar(ref scalar) => { + let wrapper = scalar.debug(quote!(ScalarWrapper)); + quote! { + { + #wrapper + ScalarWrapper(&#ident) + } + } + } + Field::Map(ref map) => { + let wrapper = map.debug(quote!(MapWrapper)); + quote! { + { + #wrapper + MapWrapper(&#ident) + } + } + } + _ => quote!(&#ident), + } + } + + pub fn methods(&self, ident: &Ident) -> Option { + match *self { + Field::Scalar(ref scalar) => scalar.methods(ident), + Field::Map(ref map) => map.methods(ident), + _ => None, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Label { + /// An optional field. + Optional, + /// A required field. + Required, + /// A repeated field. + Repeated, +} + +impl Label { + fn as_str(self) -> &'static str { + match self { + Label::Optional => "optional", + Label::Required => "required", + Label::Repeated => "repeated", + } + } + + fn variants() -> slice::Iter<'static, Label> { + const VARIANTS: &[Label] = &[Label::Optional, Label::Required, Label::Repeated]; + VARIANTS.iter() + } + + /// Parses a string into a field label. + /// If the string doesn't match a field label, `None` is returned. + fn from_attr(attr: &Meta) -> Option