Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for maps #75

Merged
merged 3 commits into from
Jan 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .npmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
; package-lock with tarball deps breaks lerna/nx - remove when https://github.com/semantic-release/github/pull/487 is merged
package-lock=false
3 changes: 2 additions & 1 deletion packages/protons/.aegir.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ export default {
build: {
config: {
platform: 'node'
}
},
bundle: false
}
}
4 changes: 3 additions & 1 deletion packages/protons/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,12 @@ It does have one or two differences:
2. All 64 bit values are represented as `BigInt`s and not `Long`s (e.g. `int64`, `uint64`, `sint64` etc)
3. Unset `optional` fields are set on the deserialized object forms as `undefined` instead of the default values
4. `singular` fields set to default values are not serialized and are set to default values when deserialized if not set - protobuf.js [diverges from the language guide](https://github.com/protobufjs/protobuf.js/issues/1468#issuecomment-745177012) around this feature
5. `map` fields can have keys of any type - protobufs.js [only supports strings](https://github.com/protobufjs/protobuf.js/issues/1203#issuecomment-488637338)
6. `map` fields are deserialized as ES6 `Map`s - protobuf.js uses `Object`s

## Missing features

Some features are missing `OneOf`, `Map`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.
Some features are missing `OneOf`s, etc due to them not being needed so far in ipfs/libp2p. If these features are important to you, please open PRs implementing them along with tests comparing the generated bytes to `protobuf.js` and `pbjs`.

## License

Expand Down
129 changes: 103 additions & 26 deletions packages/protons/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ function findDef (typeName: string, classDef: MessageDef, moduleDef: ModuleDef):
function createDefaultObject (fields: Record<string, FieldDef>, messageDef: MessageDef, moduleDef: ModuleDef): string {
const output = Object.entries(fields)
.map(([name, fieldDef]) => {
if (fieldDef.map) {
return `${name}: new Map<${types[fieldDef.keyType ?? 'string']}, ${types[fieldDef.valueType]}>()`
}

if (fieldDef.repeated) {
return `${name}: []`
}
Expand Down Expand Up @@ -280,10 +284,17 @@ interface FieldDef {
repeated: boolean
message: boolean
enum: boolean
map: boolean
valueType: string
keyType: string
}

function defineFields (fields: Record<string, FieldDef>, messageDef: MessageDef, moduleDef: ModuleDef): string[] {
return Object.entries(fields).map(([fieldName, fieldDef]) => {
if (fieldDef.map) {
return `${fieldName}: Map<${findTypeName(fieldDef.keyType ?? 'string', messageDef, moduleDef)}, ${findTypeName(fieldDef.valueType, messageDef, moduleDef)}>`
}

return `${fieldName}${fieldDef.optional ? '?' : ''}: ${findTypeName(fieldDef.type, messageDef, moduleDef)}${fieldDef.repeated ? '[]' : ''}`
})
}
Expand Down Expand Up @@ -365,7 +376,7 @@ export interface ${messageDef.name} {
${Object.entries(fields)
.map(([name, fieldDef]) => {
let codec: string = encoders[fieldDef.type]
let type: string = fieldDef.type
let type: string = fieldDef.map ? 'message' : fieldDef.type
let typeName: string = ''

if (codec == null) {
Expand All @@ -383,8 +394,10 @@ ${Object.entries(fields)

let valueTest = `obj.${name} != null`

// proto3 singular fields should only be written out if they are not the default value
if (!fieldDef.optional && !fieldDef.repeated) {
if (fieldDef.map) {
valueTest = `obj.${name} != null && obj.${name}.size !== 0`
} else if (!fieldDef.optional && !fieldDef.repeated) {
// proto3 singular fields should only be written out if they are not the default value
if (defaultValueTestGenerators[type] != null) {
valueTest = `opts.writeDefaults === true || ${defaultValueTestGenerators[type](`obj.${name}`)}`
} else if (type === 'enum') {
Expand Down Expand Up @@ -413,10 +426,11 @@ ${Object.entries(fields)
let writeField = createWriteField(`obj.${name}`)

if (fieldDef.repeated) {
writeField = `
for (const value of obj.${name}) {
if (fieldDef.map) {
writeField = `
for (const [key, value] of obj.${name}.entries()) {
${
createWriteField('value')
createWriteField('{ key, value }')
.split('\n')
.map(s => {
const trimmed = s.trim()
Expand All @@ -425,8 +439,24 @@ ${Object.entries(fields)
})
.join('\n')
}
}
`.trim()
} else {
writeField = `
for (const value of obj.${name}) {
${
createWriteField('value')
.split('\n')
.map(s => {
const trimmed = s.trim()

return trimmed === '' ? trimmed : ` ${s}`
})
.join('\n')
}
}
`.trim()
}
}

return `
Expand All @@ -448,30 +478,46 @@ ${Object.entries(fields)

switch (tag >>> 3) {
${Object.entries(fields)
.map(([name, fieldDef]) => {
let codec: string = encoders[fieldDef.type]
let type: string = fieldDef.type

if (codec == null) {
if (fieldDef.enum) {
moduleDef.imports.add('enumeration')
type = 'enum'
} else {
moduleDef.imports.add('message')
type = 'message'
.map(([fieldName, fieldDef]) => {
function createReadField (fieldName: string, fieldDef: FieldDef): string {
let codec: string = encoders[fieldDef.type]
let type: string = fieldDef.type

if (codec == null) {
if (fieldDef.enum) {
moduleDef.imports.add('enumeration')
type = 'enum'
} else {
moduleDef.imports.add('message')
type = 'message'
}

const typeName = findTypeName(fieldDef.type, messageDef, moduleDef)
codec = `${typeName}.codec()`
}

const typeName = findTypeName(fieldDef.type, messageDef, moduleDef)
codec = `${typeName}.codec()`
}
const parseValue = `${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type]()}`

if (fieldDef.map) {
return `case ${fieldDef.id}: {
const entry = ${parseValue}
obj.${fieldName}.set(entry.key, entry.value)
break
}`
} else if (fieldDef.repeated) {
return `case ${fieldDef.id}:
obj.${fieldName}.push(${parseValue})
break`
}

return `case ${fieldDef.id}:${fieldDef.rule === 'repeated'
? `
obj.${name}.push(${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type]()})`
: `
obj.${name} = ${decoderGenerators[type] == null ? `${codec}.decode(reader${type === 'message' ? ', reader.uint32()' : ''})` : decoderGenerators[type]()}`}
return `case ${fieldDef.id}:
obj.${fieldName} = ${parseValue}
break`
}).join('\n ')}
}

return createReadField(fieldName, fieldDef)
})
.join('\n ')}
default:
reader.skipType(tag & 7)
break
Expand Down Expand Up @@ -543,6 +589,7 @@ function defineModule (def: ClassDef): ModuleDef {
const fieldDef = classDef.fields[name]
fieldDef.repeated = fieldDef.rule === 'repeated'
fieldDef.optional = !fieldDef.repeated && fieldDef.options?.proto3_optional === true
fieldDef.map = fieldDef.keyType != null
}
}

Expand Down Expand Up @@ -598,6 +645,36 @@ export async function generate (source: string, flags: Flags): Promise<void> {
}

const def = JSON.parse(json)

for (const [className, classDef] of Object.entries<any>(def.nested)) {
for (const [fieldName, fieldDef] of Object.entries<any>(classDef.fields ?? {})) {
if (fieldDef.keyType == null) {
continue
}

// https://developers.google.com/protocol-buffers/docs/proto3#backwards_compatibility
const mapEntryType = `${className}$${fieldName}Entry`

classDef.nested = classDef.nested ?? {}
classDef.nested[mapEntryType] = {
fields: {
key: {
type: fieldDef.keyType,
id: 1
},
value: {
type: fieldDef.type,
id: 2
}
}
}

fieldDef.valueType = fieldDef.type
fieldDef.type = mapEntryType
fieldDef.rule = 'repeated'
}
}

const moduleDef = defineModule(def)

let lines = [
Expand Down
12 changes: 12 additions & 0 deletions packages/protons/test/fixtures/maps.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
syntax = "proto3";

message SubMessage {
string foo = 1;
}

message MapTypes {
map<string, string> stringMap = 1;
map<int32, int32> intMap = 2;
map<bool, bool> boolMap = 3;
map<string, SubMessage> messageMap = 4;
}
Loading