From 942e050bc0c14d3c8177da218f178d8dab1c0333 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 31 Jan 2024 11:42:00 +0100 Subject: [PATCH] chore: update publish config --- .github/workflows/js-test-and-release.yml | 3 +- .npmrc | 2 - README.md | 24 +---- package.json | 103 +++++++++++++++++-- packages/protons-runtime/README.md | 21 ++-- packages/protons-runtime/package.json | 92 +---------------- packages/protons-runtime/src/index.ts | 12 +++ packages/protons/README.md | 33 +++--- packages/protons/package.json | 92 +---------------- packages/protons/src/index.ts | 117 ++++++++++++++++++++++ typedoc.json | 7 ++ 11 files changed, 275 insertions(+), 231 deletions(-) delete mode 100644 .npmrc create mode 100644 typedoc.json diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 35d87d1..1f04eb0 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -9,7 +9,8 @@ on: permissions: contents: write - packages: write + id-token: write + pull-requests: write concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} diff --git a/.npmrc b/.npmrc deleted file mode 100644 index c5ebf5e..0000000 --- a/.npmrc +++ /dev/null @@ -1,2 +0,0 @@ -; package-lock with tarball deps breaks lerna/nx - remove when https://github.com/semantic-release/github/pull/487 is merged -package-lock=false diff --git a/README.md b/README.md index 9164037..e52a0cb 100644 --- a/README.md +++ b/README.md @@ -7,40 +7,26 @@ > Protobuf to ts transpiler -## Table of contents +`protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/). -- [Structure](#structure) -- [Packages](#packages) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) - -## Structure - -- [`/packages/protons`](./packages/protons) Protobuf to ts transpiler -- [`/packages/protons-benchmark`](./packages/protons-benchmark) Protobuf to ts transpiler -- [`/packages/protons-runtime`](./packages/protons-runtime) Shared code to make your bundle smaller when running protons in your app - -Transpiles `.proto` files to `.ts` - uses `Uint8Array` for `byte` fields and `BigInt` for `int64`/`uint64` and `sint64`. - -## Packages +# Packages - [`/packages/protons`](./packages/protons) The transpiler - [`/packages/protons-benchmark`](./packages/protons-benchmark) A benchmark suite - [`/packages/protons-runtime`](./packages/protons-runtime) Shared components that turn values to bytes and back again -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/protons/issues). diff --git a/package.json b/package.json index 2004ae0..5b83523 100644 --- a/package.json +++ b/package.json @@ -12,12 +12,12 @@ "url": "https://github.com/ipfs/protons/issues" }, "keywords": [ - "interface", - "libp2p" + "protobuf", + "protons" ], "private": true, "scripts": { - "reset": "aegir run clean && aegir clean ./node_modules ./package-lock.json packages/*/node_modules packages/*/package-lock.json packages/*/dist", + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", "test": "aegir run test", "test:node": "aegir run test:node", "test:chrome": "aegir run test:chrome", @@ -27,16 +27,105 @@ "test:electron-main": "aegir run test:electron-main", "test:electron-renderer": "aegir run test:electron-renderer", "clean": "aegir run clean", + "generate": "aegir run generate", "build": "aegir run build", "lint": "aegir run lint", "dep-check": "aegir run dep-check", - "release": "aegir run release", - "docs": "aegir docs" + "release": "run-s build docs:no-publish npm:release docs", + "npm:release": "aegir release", + "docs": "aegir docs", + "docs:no-publish": "aegir docs --publish false" }, "dependencies": { - "aegir": "^42.0.1" + "aegir": "^42.0.1", + "npm-run-all": "^4.1.5" }, "workspaces": [ "packages/*" - ] + ], + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + } } diff --git a/packages/protons-runtime/README.md b/packages/protons-runtime/README.md index 7ac1608..59c71cd 100644 --- a/packages/protons-runtime/README.md +++ b/packages/protons-runtime/README.md @@ -7,14 +7,17 @@ > Shared code to make your bundle smaller when running protons in your app -## Table of contents +# About -- [Install](#install) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) +This module contains serialization/deserialization code used when encoding/decoding protobufs. -## Install +It should be declared as a dependency of your project: + +```console +npm i protons-runtime +``` + +# Install ```console $ npm i protons-runtime @@ -22,18 +25,18 @@ $ npm i protons-runtime Contains shared code to reduce code duplication between modules transpiled by protons. -## API Docs +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/protons/issues). diff --git a/packages/protons-runtime/package.json b/packages/protons-runtime/package.json index 09dc9e2..49f284a 100644 --- a/packages/protons-runtime/package.json +++ b/packages/protons-runtime/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/protons/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "type": "module", "types": "./dist/src/index.d.ts", "files": [ @@ -33,97 +37,11 @@ "sourceType": "module" } }, - "release": { - "branches": [ - "main" - ], - "plugins": [ - [ - "@semantic-release/commit-analyzer", - { - "preset": "conventionalcommits", - "releaseRules": [ - { - "breaking": true, - "release": "major" - }, - { - "revert": true, - "release": "patch" - }, - { - "type": "feat", - "release": "minor" - }, - { - "type": "fix", - "release": "patch" - }, - { - "type": "docs", - "release": "patch" - }, - { - "type": "test", - "release": "patch" - }, - { - "type": "deps", - "release": "patch" - }, - { - "scope": "no-release", - "release": false - } - ] - } - ], - [ - "@semantic-release/release-notes-generator", - { - "preset": "conventionalcommits", - "presetConfig": { - "types": [ - { - "type": "feat", - "section": "Features" - }, - { - "type": "fix", - "section": "Bug Fixes" - }, - { - "type": "chore", - "section": "Trivial Changes" - }, - { - "type": "docs", - "section": "Documentation" - }, - { - "type": "deps", - "section": "Dependencies" - }, - { - "type": "test", - "section": "Tests" - } - ] - } - } - ], - "@semantic-release/changelog", - "@semantic-release/npm", - "@semantic-release/github", - "@semantic-release/git" - ] - }, "scripts": { "clean": "aegir clean", "lint": "aegir lint", "dep-check": "aegir dep-check", - "build": "aegir build", - "release": "aegir release" + "build": "aegir build" }, "dependencies": { "uint8-varint": "^2.0.2", diff --git a/packages/protons-runtime/src/index.ts b/packages/protons-runtime/src/index.ts index d8c6de2..8c61d59 100644 --- a/packages/protons-runtime/src/index.ts +++ b/packages/protons-runtime/src/index.ts @@ -1,3 +1,15 @@ +/** + * @packageDocumentation + * + * This module contains serialization/deserialization code used when encoding/decoding protobufs. + * + * It should be declared as a dependency of your project: + * + * ```console + * npm i protons-runtime + * ``` + */ + import type { Codec } from './codec.js' export interface FieldDef { diff --git a/packages/protons/README.md b/packages/protons/README.md index d2b5683..d543327 100644 --- a/packages/protons/README.md +++ b/packages/protons/README.md @@ -7,26 +7,15 @@ > Protobuf to ts transpiler -## Table of contents +# About -- [Install](#install) -- [Usage](#usage) -- [Differences from protobuf.js](#differences-from-protobufjs) -- [Extra features](#extra-features) - - [Limiting the size of repeated/map elements](#limiting-the-size-of-repeatedmap-elements) - - [Overriding 64 bit types](#overriding-64-bit-types) -- [Missing features](#missing-features) -- [API Docs](#api-docs) -- [License](#license) -- [Contribute](#contribute) +`protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/). -## Install +It transpiles code to TypeScript and supports BigInts for 64 bit types. -```console -$ npm i protons -``` +The `protons` module contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution. -`protons` contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution. +Please ensure you declare them as the correct type of dependencies: ```console $ npm install --save-dev protons @@ -133,18 +122,24 @@ console.info(typeof message.stringField) // string Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`. -## API Docs +# Install + +```console +$ npm i protons +``` + +# API Docs - -## License +# License Licensed under either of - Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) - MIT ([LICENSE-MIT](LICENSE-MIT) / ) -## Contribute +# Contribute Contributions welcome! Please check out [the issues](https://github.com/ipfs/protons/issues). diff --git a/packages/protons/package.json b/packages/protons/package.json index 40758c6..c5e1e80 100644 --- a/packages/protons/package.json +++ b/packages/protons/package.json @@ -11,6 +11,10 @@ "bugs": { "url": "https://github.com/ipfs/protons/issues" }, + "publishConfig": { + "access": "public", + "provenance": true + }, "bin": { "protons": "./dist/bin/protons.js" }, @@ -35,99 +39,13 @@ "sourceType": "module" } }, - "release": { - "branches": [ - "main" - ], - "plugins": [ - [ - "@semantic-release/commit-analyzer", - { - "preset": "conventionalcommits", - "releaseRules": [ - { - "breaking": true, - "release": "major" - }, - { - "revert": true, - "release": "patch" - }, - { - "type": "feat", - "release": "minor" - }, - { - "type": "fix", - "release": "patch" - }, - { - "type": "docs", - "release": "patch" - }, - { - "type": "test", - "release": "patch" - }, - { - "type": "deps", - "release": "patch" - }, - { - "scope": "no-release", - "release": false - } - ] - } - ], - [ - "@semantic-release/release-notes-generator", - { - "preset": "conventionalcommits", - "presetConfig": { - "types": [ - { - "type": "feat", - "section": "Features" - }, - { - "type": "fix", - "section": "Bug Fixes" - }, - { - "type": "chore", - "section": "Trivial Changes" - }, - { - "type": "docs", - "section": "Documentation" - }, - { - "type": "deps", - "section": "Dependencies" - }, - { - "type": "test", - "section": "Tests" - } - ] - } - } - ], - "@semantic-release/changelog", - "@semantic-release/npm", - "@semantic-release/github", - "@semantic-release/git" - ] - }, "scripts": { "clean": "aegir clean", "lint": "aegir lint", "dep-check": "aegir dep-check", "build": "aegir build", "test": "aegir test -t node", - "test:node": "aegir test -t node --cov", - "release": "aegir release" + "test:node": "aegir test -t node --cov" }, "dependencies": { "meow": "^13.1.0", diff --git a/packages/protons/src/index.ts b/packages/protons/src/index.ts index 5b073ab..21a626e 100644 --- a/packages/protons/src/index.ts +++ b/packages/protons/src/index.ts @@ -1,5 +1,122 @@ /* eslint-disable max-depth */ +/** + * @packageDocumentation + * + * `protons` is a high performance implementation of [Protocol Buffers v3](https://protobuf.dev/programming-guides/proto3/). + * + * It transpiles code to TypeScript and supports BigInts for 64 bit types. + * + * The `protons` module contains the code to compile `.proto` files to `.ts` files and `protons-runtime` contains the code to do serialization/deserialization to `Uint8Array`s during application execution. + * + * Please ensure you declare them as the correct type of dependencies: + * + * ```console + * $ npm install --save-dev protons + * $ npm install --save protons-runtime + * ``` + * + * ## Usage + * + * First generate your `.ts` files: + * + * ```console + * $ protons ./path/to/foo.proto ./path/to/output.ts + * ``` + * + * Then run tsc over them as normal: + * + * ```console + * $ tsc + * ``` + * + * In your code import the generated classes and use them to transform to/from bytes: + * + * ```js + * import { Foo } from './foo.js' + * + * const foo = { + * message: 'hello world' + * } + * + * const encoded = Foo.encode(foo) + * const decoded = Foo.decode(encoded) + * + * console.info(decoded.message) + * // 'hello world' + * ``` + * + * ## Differences from protobuf.js + * + * This module uses the internal reader/writer from `protobuf.js` as it is highly optimised and there's no point reinventing the wheel. + * + * It does have one or two differences: + * + * 1. Supports `proto3` semantics only + * 2. All 64 bit values are represented as `BigInt`s and not `Long`s (e.g. `int64`, `uint64`, `sint64` etc) + * 3. Unset `optional` fields are set on the deserialized object forms as `undefined` instead of the default values + * 4. `singular` fields set to default values are not serialized and are set to default values when deserialized if not set - protobuf.js [diverges from the language guide](https://github.com/protobufjs/protobuf.js/issues/1468#issuecomment-745177012) around this feature + * 5. `map` fields can have keys of any type - protobufs.js [only supports strings](https://github.com/protobufjs/protobuf.js/issues/1203#issuecomment-488637338) + * 6. `map` fields are deserialized as ES6 `Map`s - protobuf.js uses `Object`s + * + * ## Extra features + * + * ### Limiting the size of repeated/map elements + * + * To protect decoders from malicious payloads, it's possible to limit the maximum size of repeated/map elements. + * + * You can either do this at compile time by using the [protons.options](https://github.com/protocolbuffers/protobuf/blob/6f1d88107f268b8ebdad6690d116e74c403e366e/docs/options.md?plain=1#L490-L493) extension: + * + * ```protobuf + * message MyMessage { + * // repeatedField cannot have more than 10 entries + * repeated uint32 repeatedField = 1 [(protons.options).limit = 10]; + * + * // stringMap cannot have more than 10 keys + * map stringMap = 2 [(protons.options).limit = 10]; + * } + * ``` + * + * Or at runtime by passing objects to the `.decode` function of your message: + * + * ```TypeScript + * const message = MyMessage.decode(buf, { + * limits: { + * repeatedField: 10, + * stringMap: 10 + * } + * }) + * ``` + * + * ### Overriding 64 bit types + * + * By default 64 bit types are implemented as [BigInt](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt)s. + * + * Sometimes this is undesirable due to [performance issues](https://betterprogramming.pub/the-downsides-of-bigints-in-javascript-6350fd807d) or code legibility. + * + * It's possible to override the JavaScript type 64 bit fields will deserialize to: + * + * ```protobuf + * message MyMessage { + * repeated int64 bigintField = 1; + * repeated int64 numberField = 2 [jstype = JS_NUMBER]; + * repeated int64 stringField = 3 [jstype = JS_STRING]; + * } + * ``` + * + * ```TypeScript + * const message = MyMessage.decode(buf) + * + * console.info(typeof message.bigintField) // bigint + * console.info(typeof message.numberField) // number + * console.info(typeof message.stringField) // string + * ``` + * + * ## Missing features + * + * Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`. + */ + import fs from 'fs/promises' import path from 'path' import { promisify } from 'util' diff --git a/typedoc.json b/typedoc.json new file mode 100644 index 0000000..79f1bd7 --- /dev/null +++ b/typedoc.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://typedoc.org/schema.json", + "name": "Protons", + "exclude": [ + "packages/protons-benchmark" + ] +}