From 60e9b1096e72f4dd9c4413768ec8e4add7a34186 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Tue, 2 Jan 2024 15:32:24 +0800 Subject: [PATCH] add event sequence id Signed-off-by: Wei Liu --- cloudevents/generic/types/types.go | 14 + cloudevents/work/agent/codec/manifest.go | 10 +- cloudevents/work/agent/codec/manifest_test.go | 23 +- .../work/agent/codec/manifestbundle.go | 12 +- .../work/agent/codec/manifestbundle_test.go | 22 +- go.mod | 1 + go.sum | 2 + test/integration/cloudevents/agent/agent.go | 8 +- .../github.com/bwmarrin/snowflake/.travis.yml | 12 + vendor/github.com/bwmarrin/snowflake/LICENSE | 23 ++ .../github.com/bwmarrin/snowflake/README.md | 143 +++++++ .../bwmarrin/snowflake/snowflake.go | 365 ++++++++++++++++++ vendor/modules.txt | 3 + 13 files changed, 624 insertions(+), 14 deletions(-) create mode 100644 vendor/github.com/bwmarrin/snowflake/.travis.yml create mode 100644 vendor/github.com/bwmarrin/snowflake/LICENSE create mode 100644 vendor/github.com/bwmarrin/snowflake/README.md create mode 100644 vendor/github.com/bwmarrin/snowflake/snowflake.go diff --git a/cloudevents/generic/types/types.go b/cloudevents/generic/types/types.go index d3453f5fa..d59f04876 100644 --- a/cloudevents/generic/types/types.go +++ b/cloudevents/generic/types/types.go @@ -48,6 +48,10 @@ const ( // ExtensionResourceVersion is the cloud event extension key of the resource version. ExtensionResourceVersion = "resourceversion" + // ExtensionSequenceID is the cloud event extension key of the event sequence ID. + // The event sequence id represents the order in which events occur on the agent/source. + ExtensionSequenceNumber = "sequenceid" + // ExtensionDeletionTimestamp is the cloud event extension key of the deletion timestamp. ExtensionDeletionTimestamp = "deletiontimestamp" @@ -159,6 +163,7 @@ type EventBuilder struct { clusterName string originalSource string resourceID string + sequenceID string resourceVersion *int64 eventType CloudEventsType deletionTimestamp time.Time @@ -181,6 +186,11 @@ func (b *EventBuilder) WithResourceVersion(resourceVersion int64) *EventBuilder return b } +func (b *EventBuilder) WithSequenceID(sequenceID string) *EventBuilder { + b.sequenceID = sequenceID + return b +} + func (b *EventBuilder) WithClusterName(clusterName string) *EventBuilder { b.clusterName = clusterName return b @@ -211,6 +221,10 @@ func (b *EventBuilder) NewEvent() cloudevents.Event { evt.SetExtension(ExtensionResourceVersion, *b.resourceVersion) } + if len(b.sequenceID) != 0 { + evt.SetExtension(ExtensionSequenceNumber, b.sequenceID) + } + if len(b.clusterName) != 0 { evt.SetExtension(ExtensionClusterName, b.clusterName) } diff --git a/cloudevents/work/agent/codec/manifest.go b/cloudevents/work/agent/codec/manifest.go index 515bb1962..15a28fdc0 100644 --- a/cloudevents/work/agent/codec/manifest.go +++ b/cloudevents/work/agent/codec/manifest.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" + "github.com/bwmarrin/snowflake" cloudevents "github.com/cloudevents/sdk-go/v2" cloudeventstypes "github.com/cloudevents/sdk-go/v2/types" @@ -29,12 +30,14 @@ const ( // ManifestCodec is a codec to encode/decode a ManifestWork/cloudevent with ManifestBundle for an agent. type ManifestCodec struct { - restMapper meta.RESTMapper + sequenceGenerator *snowflake.Node + restMapper meta.RESTMapper } -func NewManifestCodec(restMapper meta.RESTMapper) *ManifestCodec { +func NewManifestCodec(sequenceGenerator *snowflake.Node, restMapper meta.RESTMapper) *ManifestCodec { return &ManifestCodec{ - restMapper: restMapper, + sequenceGenerator: sequenceGenerator, + restMapper: restMapper, } } @@ -65,6 +68,7 @@ func (c *ManifestCodec) Encode(source string, eventType types.CloudEventsType, w evt := types.NewEventBuilder(source, eventType). WithResourceID(string(work.UID)). + WithSequenceID(c.sequenceGenerator.Generate().String()). WithResourceVersion(resourceVersion). WithClusterName(work.Namespace). WithOriginalSource(originalSource). diff --git a/cloudevents/work/agent/codec/manifest_test.go b/cloudevents/work/agent/codec/manifest_test.go index c338f1118..82cc19c50 100644 --- a/cloudevents/work/agent/codec/manifest_test.go +++ b/cloudevents/work/agent/codec/manifest_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/bwmarrin/snowflake" cloudevents "github.com/cloudevents/sdk-go/v2" corev1 "k8s.io/api/core/v1" @@ -17,7 +18,12 @@ import ( ) func TestManifestEventDataType(t *testing.T) { - codec := NewManifestCodec(nil) + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + t.Fatal(err) + } + + codec := NewManifestCodec(sequenceGenerator, nil) if codec.EventDataType() != payload.ManifestEventDataType { t.Errorf("unexpected event data type %s", codec.EventDataType()) @@ -141,9 +147,14 @@ func TestManifestEncode(t *testing.T) { }, } + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + t.Fatal(err) + } + for _, c := range cases { t.Run(c.name, func(t *testing.T) { - codec := NewManifestCodec(nil) + codec := NewManifestCodec(sequenceGenerator, nil) _, err := codec.Encode("cluster1-work-agent", c.eventType, c.work) if c.expectedErr { @@ -281,10 +292,14 @@ func TestManifestDecode(t *testing.T) { }, } + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + t.Fatal(err) + } + for _, c := range cases { t.Run(c.name, func(t *testing.T) { - codec := NewManifestCodec(nil) - + codec := NewManifestCodec(sequenceGenerator, nil) _, err := codec.Decode(c.event) if c.expectedErr { if err == nil { diff --git a/cloudevents/work/agent/codec/manifestbundle.go b/cloudevents/work/agent/codec/manifestbundle.go index 7eb10b668..140a50c0a 100644 --- a/cloudevents/work/agent/codec/manifestbundle.go +++ b/cloudevents/work/agent/codec/manifestbundle.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" + "github.com/bwmarrin/snowflake" cloudevents "github.com/cloudevents/sdk-go/v2" cloudeventstypes "github.com/cloudevents/sdk-go/v2/types" @@ -17,10 +18,14 @@ import ( ) // ManifestBundleCodec is a codec to encode/decode a ManifestWork/cloudevent with ManifestBundle for an agent. -type ManifestBundleCodec struct{} +type ManifestBundleCodec struct { + sequenceGenerator *snowflake.Node +} -func NewManifestBundleCodec() *ManifestBundleCodec { - return &ManifestBundleCodec{} +func NewManifestBundleCodec(sequenceGenerator *snowflake.Node) *ManifestBundleCodec { + return &ManifestBundleCodec{ + sequenceGenerator: sequenceGenerator, + } } // EventDataType always returns the event data type `io.open-cluster-management.works.v1alpha1.manifestbundles`. @@ -46,6 +51,7 @@ func (c *ManifestBundleCodec) Encode(source string, eventType types.CloudEventsT evt := types.NewEventBuilder(source, eventType). WithResourceID(string(work.UID)). + WithSequenceID(c.sequenceGenerator.Generate().String()). WithResourceVersion(resourceVersion). WithClusterName(work.Namespace). WithOriginalSource(originalSource). diff --git a/cloudevents/work/agent/codec/manifestbundle_test.go b/cloudevents/work/agent/codec/manifestbundle_test.go index 4306593bd..8101eb92b 100644 --- a/cloudevents/work/agent/codec/manifestbundle_test.go +++ b/cloudevents/work/agent/codec/manifestbundle_test.go @@ -3,6 +3,7 @@ package codec import ( "testing" + "github.com/bwmarrin/snowflake" cloudevents "github.com/cloudevents/sdk-go/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,7 +15,12 @@ import ( ) func TestManifestBundleEventDataType(t *testing.T) { - codec := NewManifestBundleCodec() + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + t.Fatal(err) + } + + codec := NewManifestBundleCodec(sequenceGenerator) if codec.EventDataType() != payload.ManifestBundleEventDataType { t.Errorf("unexpected event data type %s", codec.EventDataType()) @@ -90,9 +96,14 @@ func TestManifestBundleEncode(t *testing.T) { }, } + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + t.Fatal(err) + } + for _, c := range cases { t.Run(c.name, func(t *testing.T) { - codec := NewManifestBundleCodec() + codec := NewManifestBundleCodec(sequenceGenerator) _, err := codec.Encode("cluster1-work-agent", c.eventType, c.work) if c.expectedErr { @@ -229,9 +240,14 @@ func TestManifestBundleDecode(t *testing.T) { }, } + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + t.Fatal(err) + } + for _, c := range cases { t.Run(c.name, func(t *testing.T) { - codec := NewManifestBundleCodec() + codec := NewManifestBundleCodec(sequenceGenerator) _, err := codec.Decode(c.event) if c.expectedErr { diff --git a/go.mod b/go.mod index 33bdc2fca..9c7e7e018 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module open-cluster-management.io/api go 1.20 require ( + github.com/bwmarrin/snowflake v0.3.0 github.com/cloudevents/sdk-go/protocol/mqtt_paho/v2 v2.0.0-20231030012137-0836a524e995 github.com/cloudevents/sdk-go/v2 v2.14.0 github.com/eclipse/paho.golang v0.11.0 diff --git a/go.sum b/go.sum index 4678e7ad0..81c7b513e 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudevents/sdk-go/protocol/mqtt_paho/v2 v2.0.0-20231030012137-0836a524e995 h1:pXyRKZ0T5WoB6X9QnHS5cEyW0Got39bNQIECxGUKVO4= diff --git a/test/integration/cloudevents/agent/agent.go b/test/integration/cloudevents/agent/agent.go index 6d372cbda..446c6e151 100644 --- a/test/integration/cloudevents/agent/agent.go +++ b/test/integration/cloudevents/agent/agent.go @@ -3,15 +3,21 @@ package agent import ( "context" + "github.com/bwmarrin/snowflake" "open-cluster-management.io/api/cloudevents/generic/options/mqtt" "open-cluster-management.io/api/cloudevents/work" "open-cluster-management.io/api/cloudevents/work/agent/codec" ) func StartWorkAgent(ctx context.Context, clusterName string, config *mqtt.MQTTOptions) (*work.ClientHolder, error) { + sequenceGenerator, err := snowflake.NewNode(1) + if err != nil { + return nil, err + } + clientHolder, err := work.NewClientHolderBuilder(clusterName, config). WithClusterName(clusterName). - WithCodecs(codec.NewManifestCodec(nil)). + WithCodecs(codec.NewManifestCodec(sequenceGenerator, nil)). NewClientHolder(ctx) if err != nil { return nil, err diff --git a/vendor/github.com/bwmarrin/snowflake/.travis.yml b/vendor/github.com/bwmarrin/snowflake/.travis.yml new file mode 100644 index 000000000..75d9f5c24 --- /dev/null +++ b/vendor/github.com/bwmarrin/snowflake/.travis.yml @@ -0,0 +1,12 @@ +language: go +go: + - 1.11.x + - 1.12.x +install: + - go get -v . + - go get -v golang.org/x/lint/golint +script: + - diff <(gofmt -d .) <(echo -n) + - go vet -x ./... + - golint -set_exit_status ./... + - go test -v -race ./... diff --git a/vendor/github.com/bwmarrin/snowflake/LICENSE b/vendor/github.com/bwmarrin/snowflake/LICENSE new file mode 100644 index 000000000..ef39145e5 --- /dev/null +++ b/vendor/github.com/bwmarrin/snowflake/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2016, Bruce +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/bwmarrin/snowflake/README.md b/vendor/github.com/bwmarrin/snowflake/README.md new file mode 100644 index 000000000..23af210b6 --- /dev/null +++ b/vendor/github.com/bwmarrin/snowflake/README.md @@ -0,0 +1,143 @@ +snowflake +==== +[![GoDoc](https://godoc.org/github.com/bwmarrin/snowflake?status.svg)](https://godoc.org/github.com/bwmarrin/snowflake) [![Go report](http://goreportcard.com/badge/bwmarrin/snowflake)](http://goreportcard.com/report/bwmarrin/snowflake) [![Coverage](http://gocover.io/_badge/github.com/bwmarrin/snowflake)](https://gocover.io/github.com/bwmarrin/snowflake) [![Build Status](https://travis-ci.org/bwmarrin/snowflake.svg?branch=master)](https://travis-ci.org/bwmarrin/snowflake) [![Discord Gophers](https://img.shields.io/badge/Discord%20Gophers-%23info-blue.svg)](https://discord.gg/0f1SbxBZjYq9jLBk) + +snowflake is a [Go](https://golang.org/) package that provides +* A very simple Twitter snowflake generator. +* Methods to parse existing snowflake IDs. +* Methods to convert a snowflake ID into several other data types and back. +* JSON Marshal/Unmarshal functions to easily use snowflake IDs within a JSON API. +* Monotonic Clock calculations protect from clock drift. + +**For help with this package or general Go discussion, please join the [Discord +Gophers](https://discord.gg/0f1SbxBZjYq9jLBk) chat server.** + +## Status +This package should be considered stable and completed. Any additions in the +future will strongly avoid API changes to existing functions. + +### ID Format +By default, the ID format follows the original Twitter snowflake format. +* The ID as a whole is a 63 bit integer stored in an int64 +* 41 bits are used to store a timestamp with millisecond precision, using a custom epoch. +* 10 bits are used to store a node id - a range from 0 through 1023. +* 12 bits are used to store a sequence number - a range from 0 through 4095. + +### Custom Format +You can alter the number of bits used for the node id and step number (sequence) +by setting the snowflake.NodeBits and snowflake.StepBits values. Remember that +There is a maximum of 22 bits available that can be shared between these two +values. You do not have to use all 22 bits. + +### Custom Epoch +By default this package uses the Twitter Epoch of 1288834974657 or Nov 04 2010 01:42:54. +You can set your own epoch value by setting snowflake.Epoch to a time in milliseconds +to use as the epoch. + +### Custom Notes +When setting custom epoch or bit values you need to set them prior to calling +any functions on the snowflake package, including NewNode(). Otherwise the +custom values you set will not be applied correctly. + +### How it Works. +Each time you generate an ID, it works, like this. +* A timestamp with millisecond precision is stored using 41 bits of the ID. +* Then the NodeID is added in subsequent bits. +* Then the Sequence Number is added, starting at 0 and incrementing for each ID generated in the same millisecond. If you generate enough IDs in the same millisecond that the sequence would roll over or overfill then the generate function will pause until the next millisecond. + +The default Twitter format shown below. +``` ++--------------------------------------------------------------------------+ +| 1 Bit Unused | 41 Bit Timestamp | 10 Bit NodeID | 12 Bit Sequence ID | ++--------------------------------------------------------------------------+ +``` + +Using the default settings, this allows for 4096 unique IDs to be generated every millisecond, per Node ID. +## Getting Started + +### Installing + +This assumes you already have a working Go environment, if not please see +[this page](https://golang.org/doc/install) first. + +```sh +go get github.com/bwmarrin/snowflake +``` + + +### Usage + +Import the package into your project then construct a new snowflake Node using a +unique node number. The default settings permit a node number range from 0 to 1023. +If you have set a custom NodeBits value, you will need to calculate what your +node number range will be. With the node object call the Generate() method to +generate and return a unique snowflake ID. + +Keep in mind that each node you create must have a unique node number, even +across multiple servers. If you do not keep node numbers unique the generator +cannot guarantee unique IDs across all nodes. + + +**Example Program:** + +```go +package main + +import ( + "fmt" + + "github.com/bwmarrin/snowflake" +) + +func main() { + + // Create a new Node with a Node number of 1 + node, err := snowflake.NewNode(1) + if err != nil { + fmt.Println(err) + return + } + + // Generate a snowflake ID. + id := node.Generate() + + // Print out the ID in a few different ways. + fmt.Printf("Int64 ID: %d\n", id) + fmt.Printf("String ID: %s\n", id) + fmt.Printf("Base2 ID: %s\n", id.Base2()) + fmt.Printf("Base64 ID: %s\n", id.Base64()) + + // Print out the ID's timestamp + fmt.Printf("ID Time : %d\n", id.Time()) + + // Print out the ID's node number + fmt.Printf("ID Node : %d\n", id.Node()) + + // Print out the ID's sequence number + fmt.Printf("ID Step : %d\n", id.Step()) + + // Generate and print, all in one. + fmt.Printf("ID : %d\n", node.Generate().Int64()) +} +``` + +### Performance + +With default settings, this snowflake generator should be sufficiently fast +enough on most systems to generate 4096 unique ID's per millisecond. This is +the maximum that the snowflake ID format supports. That is, around 243-244 +nanoseconds per operation. + +Since the snowflake generator is single threaded the primary limitation will be +the maximum speed of a single processor on your system. + +To benchmark the generator on your system run the following command inside the +snowflake package directory. + +```sh +go test -run=^$ -bench=. +``` + +If your curious, check out this commit that shows benchmarks that compare a few +different ways of implementing a snowflake generator in Go. +* https://github.com/bwmarrin/snowflake/tree/9befef8908df13f4102ed21f42b083dd862b5036 diff --git a/vendor/github.com/bwmarrin/snowflake/snowflake.go b/vendor/github.com/bwmarrin/snowflake/snowflake.go new file mode 100644 index 000000000..3ea0856d3 --- /dev/null +++ b/vendor/github.com/bwmarrin/snowflake/snowflake.go @@ -0,0 +1,365 @@ +// Package snowflake provides a very simple Twitter snowflake generator and parser. +package snowflake + +import ( + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "strconv" + "sync" + "time" +) + +var ( + // Epoch is set to the twitter snowflake epoch of Nov 04 2010 01:42:54 UTC in milliseconds + // You may customize this to set a different epoch for your application. + Epoch int64 = 1288834974657 + + // NodeBits holds the number of bits to use for Node + // Remember, you have a total 22 bits to share between Node/Step + NodeBits uint8 = 10 + + // StepBits holds the number of bits to use for Step + // Remember, you have a total 22 bits to share between Node/Step + StepBits uint8 = 12 + + // DEPRECATED: the below four variables will be removed in a future release. + mu sync.Mutex + nodeMax int64 = -1 ^ (-1 << NodeBits) + nodeMask = nodeMax << StepBits + stepMask int64 = -1 ^ (-1 << StepBits) + timeShift = NodeBits + StepBits + nodeShift = StepBits +) + +const encodeBase32Map = "ybndrfg8ejkmcpqxot1uwisza345h769" + +var decodeBase32Map [256]byte + +const encodeBase58Map = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ" + +var decodeBase58Map [256]byte + +// A JSONSyntaxError is returned from UnmarshalJSON if an invalid ID is provided. +type JSONSyntaxError struct{ original []byte } + +func (j JSONSyntaxError) Error() string { + return fmt.Sprintf("invalid snowflake ID %q", string(j.original)) +} + +// ErrInvalidBase58 is returned by ParseBase58 when given an invalid []byte +var ErrInvalidBase58 = errors.New("invalid base58") + +// ErrInvalidBase32 is returned by ParseBase32 when given an invalid []byte +var ErrInvalidBase32 = errors.New("invalid base32") + +// Create maps for decoding Base58/Base32. +// This speeds up the process tremendously. +func init() { + + for i := 0; i < len(encodeBase58Map); i++ { + decodeBase58Map[i] = 0xFF + } + + for i := 0; i < len(encodeBase58Map); i++ { + decodeBase58Map[encodeBase58Map[i]] = byte(i) + } + + for i := 0; i < len(encodeBase32Map); i++ { + decodeBase32Map[i] = 0xFF + } + + for i := 0; i < len(encodeBase32Map); i++ { + decodeBase32Map[encodeBase32Map[i]] = byte(i) + } +} + +// A Node struct holds the basic information needed for a snowflake generator +// node +type Node struct { + mu sync.Mutex + epoch time.Time + time int64 + node int64 + step int64 + + nodeMax int64 + nodeMask int64 + stepMask int64 + timeShift uint8 + nodeShift uint8 +} + +// An ID is a custom type used for a snowflake ID. This is used so we can +// attach methods onto the ID. +type ID int64 + +// NewNode returns a new snowflake node that can be used to generate snowflake +// IDs +func NewNode(node int64) (*Node, error) { + + // re-calc in case custom NodeBits or StepBits were set + // DEPRECATED: the below block will be removed in a future release. + mu.Lock() + nodeMax = -1 ^ (-1 << NodeBits) + nodeMask = nodeMax << StepBits + stepMask = -1 ^ (-1 << StepBits) + timeShift = NodeBits + StepBits + nodeShift = StepBits + mu.Unlock() + + n := Node{} + n.node = node + n.nodeMax = -1 ^ (-1 << NodeBits) + n.nodeMask = n.nodeMax << StepBits + n.stepMask = -1 ^ (-1 << StepBits) + n.timeShift = NodeBits + StepBits + n.nodeShift = StepBits + + if n.node < 0 || n.node > n.nodeMax { + return nil, errors.New("Node number must be between 0 and " + strconv.FormatInt(n.nodeMax, 10)) + } + + var curTime = time.Now() + // add time.Duration to curTime to make sure we use the monotonic clock if available + n.epoch = curTime.Add(time.Unix(Epoch/1000, (Epoch%1000)*1000000).Sub(curTime)) + + return &n, nil +} + +// Generate creates and returns a unique snowflake ID +// To help guarantee uniqueness +// - Make sure your system is keeping accurate system time +// - Make sure you never have multiple nodes running with the same node ID +func (n *Node) Generate() ID { + + n.mu.Lock() + + now := time.Since(n.epoch).Nanoseconds() / 1000000 + + if now == n.time { + n.step = (n.step + 1) & n.stepMask + + if n.step == 0 { + for now <= n.time { + now = time.Since(n.epoch).Nanoseconds() / 1000000 + } + } + } else { + n.step = 0 + } + + n.time = now + + r := ID((now)<= 32 { + b = append(b, encodeBase32Map[f%32]) + f /= 32 + } + b = append(b, encodeBase32Map[f]) + + for x, y := 0, len(b)-1; x < y; x, y = x+1, y-1 { + b[x], b[y] = b[y], b[x] + } + + return string(b) +} + +// ParseBase32 parses a base32 []byte into a snowflake ID +// NOTE: There are many different base32 implementations so becareful when +// doing any interoperation. +func ParseBase32(b []byte) (ID, error) { + + var id int64 + + for i := range b { + if decodeBase32Map[b[i]] == 0xFF { + return -1, ErrInvalidBase32 + } + id = id*32 + int64(decodeBase32Map[b[i]]) + } + + return ID(id), nil +} + +// Base36 returns a base36 string of the snowflake ID +func (f ID) Base36() string { + return strconv.FormatInt(int64(f), 36) +} + +// ParseBase36 converts a Base36 string into a snowflake ID +func ParseBase36(id string) (ID, error) { + i, err := strconv.ParseInt(id, 36, 64) + return ID(i), err +} + +// Base58 returns a base58 string of the snowflake ID +func (f ID) Base58() string { + + if f < 58 { + return string(encodeBase58Map[f]) + } + + b := make([]byte, 0, 11) + for f >= 58 { + b = append(b, encodeBase58Map[f%58]) + f /= 58 + } + b = append(b, encodeBase58Map[f]) + + for x, y := 0, len(b)-1; x < y; x, y = x+1, y-1 { + b[x], b[y] = b[y], b[x] + } + + return string(b) +} + +// ParseBase58 parses a base58 []byte into a snowflake ID +func ParseBase58(b []byte) (ID, error) { + + var id int64 + + for i := range b { + if decodeBase58Map[b[i]] == 0xFF { + return -1, ErrInvalidBase58 + } + id = id*58 + int64(decodeBase58Map[b[i]]) + } + + return ID(id), nil +} + +// Base64 returns a base64 string of the snowflake ID +func (f ID) Base64() string { + return base64.StdEncoding.EncodeToString(f.Bytes()) +} + +// ParseBase64 converts a base64 string into a snowflake ID +func ParseBase64(id string) (ID, error) { + b, err := base64.StdEncoding.DecodeString(id) + if err != nil { + return -1, err + } + return ParseBytes(b) + +} + +// Bytes returns a byte slice of the snowflake ID +func (f ID) Bytes() []byte { + return []byte(f.String()) +} + +// ParseBytes converts a byte slice into a snowflake ID +func ParseBytes(id []byte) (ID, error) { + i, err := strconv.ParseInt(string(id), 10, 64) + return ID(i), err +} + +// IntBytes returns an array of bytes of the snowflake ID, encoded as a +// big endian integer. +func (f ID) IntBytes() [8]byte { + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(f)) + return b +} + +// ParseIntBytes converts an array of bytes encoded as big endian integer as +// a snowflake ID +func ParseIntBytes(id [8]byte) ID { + return ID(int64(binary.BigEndian.Uint64(id[:]))) +} + +// Time returns an int64 unix timestamp in milliseconds of the snowflake ID time +// DEPRECATED: the below function will be removed in a future release. +func (f ID) Time() int64 { + return (int64(f) >> timeShift) + Epoch +} + +// Node returns an int64 of the snowflake ID node number +// DEPRECATED: the below function will be removed in a future release. +func (f ID) Node() int64 { + return int64(f) & nodeMask >> nodeShift +} + +// Step returns an int64 of the snowflake step (or sequence) number +// DEPRECATED: the below function will be removed in a future release. +func (f ID) Step() int64 { + return int64(f) & stepMask +} + +// MarshalJSON returns a json byte array string of the snowflake ID. +func (f ID) MarshalJSON() ([]byte, error) { + buff := make([]byte, 0, 22) + buff = append(buff, '"') + buff = strconv.AppendInt(buff, int64(f), 10) + buff = append(buff, '"') + return buff, nil +} + +// UnmarshalJSON converts a json byte array of a snowflake ID into an ID type. +func (f *ID) UnmarshalJSON(b []byte) error { + if len(b) < 3 || b[0] != '"' || b[len(b)-1] != '"' { + return JSONSyntaxError{b} + } + + i, err := strconv.ParseInt(string(b[1:len(b)-1]), 10, 64) + if err != nil { + return err + } + + *f = ID(i) + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 98157b001..1dcfe2515 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,6 +4,9 @@ github.com/beorn7/perks/quantile # github.com/blang/semver/v4 v4.0.0 ## explicit; go 1.14 github.com/blang/semver/v4 +# github.com/bwmarrin/snowflake v0.3.0 +## explicit +github.com/bwmarrin/snowflake # github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2