From 301c4f7bc87cb8c63d149fbdce5a73edfe324f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Thu, 23 Apr 2020 17:41:47 +0200 Subject: [PATCH 01/28] Refactor messaging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 10 +- cmd/http/main.go | 15 +- cmd/influxdb-writer/main.go | 11 +- cmd/mongodb-writer/main.go | 11 +- cmd/mqtt/main.go | 13 +- cmd/postgres-writer/main.go | 10 +- go.mod | 2 + http/adapter.go | 17 +- http/api/logging.go | 4 +- http/api/metrics.go | 4 +- http/api/requests.go | 6 +- http/api/transport.go | 11 +- http/mocks/publisher.go | 20 +- message.pb.go | 572 +++++++++++++++++++++++++ message.proto | 15 + mqtt/adapter.go | 112 +++-- nats/pubsub.go | 110 +++++ pubsub.go | 26 ++ transformers/senml/transformer.go | 15 +- transformers/senml/transformer_test.go | 35 +- transformers/service.go | 4 +- writers/writer.go | 42 +- 22 files changed, 879 insertions(+), 186 deletions(-) create mode 100644 message.pb.go create mode 100644 message.proto create mode 100644 nats/pubsub.go create mode 100644 pubsub.go diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 9b6ce3117a..7daed0b5ea 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -16,12 +16,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/gocql/gocql" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/cassandra" + broker "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -69,19 +69,19 @@ func main() { log.Fatalf(err.Error()) } - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() session := connectToCassandra(cfg.dbCfg, logger) defer session.Close() repo := newService(session, logger) st := senml.New(cfg.contentType) - if err := writers.Start(b, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Cassandra writer: %s", err)) } diff --git a/cmd/http/main.go b/cmd/http/main.go index 07596225e7..96820ba65a 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -19,11 +19,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/http/api" "github.com/mainflux/mainflux/logger" + nats "github.com/mainflux/mainflux/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" + broker "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -79,15 +80,17 @@ func main() { thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) defer thingsCloser.Close() - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() - cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - svc := adapter.New(b, cc) + b := nats.New(nc, "", "", logger) + + tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) + svc := adapter.New(b, tc) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index 75e121ff6a..3e6be292a5 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -14,12 +14,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" influxdata "github.com/influxdata/influxdb/client/v2" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/influxdb" + broker "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -70,12 +70,12 @@ func main() { log.Fatalf(err.Error()) } - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() client, err := influxdata.NewHTTPClient(clientCfg) if err != nil { @@ -90,7 +90,8 @@ func main() { repo = api.LoggingMiddleware(repo, logger) repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(b, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + + if err := writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start InfluxDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 810e02acfd..d19fe3cfe7 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -14,12 +14,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/mongodb" + broker "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -66,12 +66,12 @@ func main() { log.Fatal(err) } - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) @@ -87,7 +87,8 @@ func main() { repo = api.LoggingMiddleware(repo, logger) repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(b, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + + if err := writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start MongoDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index e40f76ffcd..0a5f5709e5 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -14,14 +14,15 @@ import ( "github.com/go-redis/redis" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" mqtt "github.com/mainflux/mainflux/mqtt" mr "github.com/mainflux/mainflux/mqtt/redis" + nats "github.com/mainflux/mainflux/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" mp "github.com/mainflux/mproxy/pkg/mqtt" "github.com/mainflux/mproxy/pkg/session" ws "github.com/mainflux/mproxy/pkg/websocket" + broker "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" jconfig "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" @@ -129,17 +130,19 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() + + b := nats.New(nc, "", "", logger) es := mr.NewEventStore(rc, cfg.instance) // Event handler for MQTT hooks - evt := mqtt.New(b, cc, es, logger, tracer) + evt := mqtt.New([]mainflux.Publisher{b}, cc, es, logger, tracer) errs := make(chan error, 2) diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index 0b0d4ad7ea..694988dc87 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -14,12 +14,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/postgres" + broker "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -75,19 +75,19 @@ func main() { log.Fatalf(err.Error()) } - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() db := connectToDB(cfg.dbConfig, logger) defer db.Close() repo := newService(db, logger) st := senml.New(cfg.contentType) - if err = writers.Start(b, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err = writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) } diff --git a/go.mod b/go.mod index 4ca03931bb..981e7c45ad 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mainflux/mproxy v0.1.8 github.com/mainflux/senml v1.0.1 + github.com/nats-io/go-nats v1.6.0 // indirect github.com/nats-io/nats.go v1.9.1 github.com/opentracing/opentracing-go v1.1.0 github.com/prometheus/client_golang v0.9.3 @@ -35,6 +36,7 @@ require ( golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b gonum.org/v1/gonum v0.0.0-20190808205415-ced62fe5104b + google.golang.org/genproto v0.0.0-20200225123651-fc8f55426688 // indirect google.golang.org/grpc v1.27.1 gopkg.in/ory/dockertest.v3 v3.3.5 ) diff --git a/http/adapter.go b/http/adapter.go index 0e66dc2156..874683fbcd 100644 --- a/http/adapter.go +++ b/http/adapter.go @@ -9,31 +9,30 @@ import ( "context" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" ) // Service specifies coap service API. type Service interface { // Publish Messssage - Publish(ctx context.Context, token string, msg broker.Message) error + Publish(ctx context.Context, token string, msg mainflux.Message) error } var _ Service = (*adapterService)(nil) type adapterService struct { - broker broker.Nats - things mainflux.ThingsServiceClient + publisher mainflux.Publisher + things mainflux.ThingsServiceClient } // New instantiates the HTTP adapter implementation. -func New(broker broker.Nats, things mainflux.ThingsServiceClient) Service { +func New(publisher mainflux.Publisher, things mainflux.ThingsServiceClient) Service { return &adapterService{ - broker: broker, - things: things, + publisher: publisher, + things: things, } } -func (as *adapterService) Publish(ctx context.Context, token string, msg broker.Message) error { +func (as *adapterService) Publish(ctx context.Context, token string, msg mainflux.Message) error { ar := &mainflux.AccessByKeyReq{ Token: token, ChanID: msg.GetChannel(), @@ -44,5 +43,5 @@ func (as *adapterService) Publish(ctx context.Context, token string, msg broker. } msg.Publisher = thid.GetValue() - return as.broker.Publish(ctx, token, msg) + return as.publisher.Publish(msg) } diff --git a/http/api/logging.go b/http/api/logging.go index 8a7df9b678..8a41d43929 100644 --- a/http/api/logging.go +++ b/http/api/logging.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/http" log "github.com/mainflux/mainflux/logger" ) @@ -27,7 +27,7 @@ func LoggingMiddleware(svc http.Service, logger log.Logger) http.Service { return &loggingMiddleware{logger, svc} } -func (lm *loggingMiddleware) Publish(ctx context.Context, token string, msg broker.Message) (err error) { +func (lm *loggingMiddleware) Publish(ctx context.Context, token string, msg mainflux.Message) (err error) { defer func(begin time.Time) { destChannel := msg.Channel if msg.Subtopic != "" { diff --git a/http/api/metrics.go b/http/api/metrics.go index d48cf5b7e6..d7b9ffb975 100644 --- a/http/api/metrics.go +++ b/http/api/metrics.go @@ -10,7 +10,7 @@ import ( "time" "github.com/go-kit/kit/metrics" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/http" ) @@ -31,7 +31,7 @@ func MetricsMiddleware(svc http.Service, counter metrics.Counter, latency metric } } -func (mm *metricsMiddleware) Publish(ctx context.Context, token string, msg broker.Message) error { +func (mm *metricsMiddleware) Publish(ctx context.Context, token string, msg mainflux.Message) error { defer func(begin time.Time) { mm.counter.With("method", "publish").Add(1) mm.latency.With("method", "publish").Observe(time.Since(begin).Seconds()) diff --git a/http/api/requests.go b/http/api/requests.go index 1733a7f3dd..4c7afc412f 100644 --- a/http/api/requests.go +++ b/http/api/requests.go @@ -3,9 +3,11 @@ package api -import "github.com/mainflux/mainflux/broker" +import ( + "github.com/mainflux/mainflux" +) type publishReq struct { - msg broker.Message + msg mainflux.Message token string } diff --git a/http/api/transport.go b/http/api/transport.go index 61e8672475..53b2dc2512 100644 --- a/http/api/transport.go +++ b/http/api/transport.go @@ -17,9 +17,7 @@ import ( kitot "github.com/go-kit/kit/tracing/opentracing" kithttp "github.com/go-kit/kit/transport/http" "github.com/go-zoo/bone" - "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/things" opentracing "github.com/opentracing/opentracing-go" @@ -110,17 +108,12 @@ func decodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { return nil, err } - created, err := ptypes.TimestampProto(time.Now()) - if err != nil { - return nil, err - } - - msg := broker.Message{ + msg := mainflux.Message{ Protocol: protocol, Channel: chanID, Subtopic: subtopic, Payload: payload, - Created: created, + Occured: time.Now().UnixNano(), } req := publishReq{ diff --git a/http/mocks/publisher.go b/http/mocks/publisher.go index 7b4908878a..86fe3f836d 100644 --- a/http/mocks/publisher.go +++ b/http/mocks/publisher.go @@ -4,30 +4,16 @@ package mocks import ( - "context" - - "github.com/mainflux/mainflux/broker" - "github.com/nats-io/nats.go" + "github.com/mainflux/mainflux" ) type mockPublisher struct{} // NewPublisher returns mock message publisher. -func NewPublisher() broker.Nats { +func NewPublisher() mainflux.Publisher { return mockPublisher{} } -func (pub mockPublisher) Publish(_ context.Context, _ string, msg broker.Message) error { +func (pub mockPublisher) Publish(mainflux.Message) error { return nil } - -func (pub mockPublisher) Subscribe(subject string, f func(*nats.Msg)) (*nats.Subscription, error) { - return nil, nil -} - -func (pub mockPublisher) QueueSubscribe(subject, queue string, f func(*nats.Msg)) (*nats.Subscription, error) { - return nil, nil -} - -func (pub mockPublisher) Close() { -} diff --git a/message.pb.go b/message.pb.go new file mode 100644 index 0000000000..4a67eb2c11 --- /dev/null +++ b/message.pb.go @@ -0,0 +1,572 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: message.proto + +package mainflux + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +// Message represents a message emitted by the Mainflux adapters layer. +type Message struct { + Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` + Subtopic string `protobuf:"bytes,2,opt,name=subtopic,proto3" json:"subtopic,omitempty"` + Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` + Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` + Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` + Occured int64 `protobuf:"varint,6,opt,name=occured,proto3" json:"occured,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} +func (*Message) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{0} +} +func (m *Message) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Message.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Message) XXX_Merge(src proto.Message) { + xxx_messageInfo_Message.Merge(m, src) +} +func (m *Message) XXX_Size() int { + return m.Size() +} +func (m *Message) XXX_DiscardUnknown() { + xxx_messageInfo_Message.DiscardUnknown(m) +} + +var xxx_messageInfo_Message proto.InternalMessageInfo + +func (m *Message) GetChannel() string { + if m != nil { + return m.Channel + } + return "" +} + +func (m *Message) GetSubtopic() string { + if m != nil { + return m.Subtopic + } + return "" +} + +func (m *Message) GetPublisher() string { + if m != nil { + return m.Publisher + } + return "" +} + +func (m *Message) GetProtocol() string { + if m != nil { + return m.Protocol + } + return "" +} + +func (m *Message) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func (m *Message) GetOccured() int64 { + if m != nil { + return m.Occured + } + return 0 +} + +func init() { + proto.RegisterType((*Message)(nil), "mainflux.Message") +} + +func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } + +var fileDescriptor_33c57e4bae7b9afd = []byte{ + // 186 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, + 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xc8, 0x4d, 0xcc, 0xcc, 0x4b, + 0xcb, 0x29, 0xad, 0x50, 0x5a, 0xcb, 0xc8, 0xc5, 0xee, 0x0b, 0x91, 0x13, 0x92, 0xe0, 0x62, 0x4f, + 0xce, 0x48, 0xcc, 0xcb, 0x4b, 0xcd, 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0x71, 0x85, + 0xa4, 0xb8, 0x38, 0x8a, 0x4b, 0x93, 0x4a, 0xf2, 0x0b, 0x32, 0x93, 0x25, 0x98, 0xc0, 0x52, 0x70, + 0xbe, 0x90, 0x0c, 0x17, 0x67, 0x41, 0x69, 0x52, 0x4e, 0x66, 0x71, 0x46, 0x6a, 0x91, 0x04, 0x33, + 0x58, 0x12, 0x21, 0x00, 0xd2, 0x09, 0xb6, 0x32, 0x39, 0x3f, 0x47, 0x82, 0x05, 0xa2, 0x13, 0xc6, + 0x07, 0xd9, 0x57, 0x90, 0x58, 0x99, 0x93, 0x9f, 0x98, 0x22, 0xc1, 0xaa, 0xc0, 0xa8, 0xc1, 0x13, + 0x04, 0xe3, 0x82, 0x64, 0xf2, 0x93, 0x93, 0x4b, 0x8b, 0x52, 0x53, 0x24, 0xd8, 0x14, 0x18, 0x35, + 0x98, 0x83, 0x60, 0x5c, 0x27, 0x81, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, + 0x48, 0x8e, 0x71, 0xc6, 0x63, 0x39, 0x86, 0x24, 0x36, 0xb0, 0x79, 0xc6, 0x80, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xe5, 0x53, 0xe6, 0x15, 0xe3, 0x00, 0x00, 0x00, +} + +func (m *Message) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Message) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Message) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Occured != 0 { + i = encodeVarintMessage(dAtA, i, uint64(m.Occured)) + i-- + dAtA[i] = 0x30 + } + if len(m.Payload) > 0 { + i -= len(m.Payload) + copy(dAtA[i:], m.Payload) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Payload))) + i-- + dAtA[i] = 0x2a + } + if len(m.Protocol) > 0 { + i -= len(m.Protocol) + copy(dAtA[i:], m.Protocol) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Protocol))) + i-- + dAtA[i] = 0x22 + } + if len(m.Publisher) > 0 { + i -= len(m.Publisher) + copy(dAtA[i:], m.Publisher) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Publisher))) + i-- + dAtA[i] = 0x1a + } + if len(m.Subtopic) > 0 { + i -= len(m.Subtopic) + copy(dAtA[i:], m.Subtopic) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Subtopic))) + i-- + dAtA[i] = 0x12 + } + if len(m.Channel) > 0 { + i -= len(m.Channel) + copy(dAtA[i:], m.Channel) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Channel))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { + offset -= sovMessage(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Message) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Channel) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + l = len(m.Subtopic) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + l = len(m.Publisher) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + l = len(m.Protocol) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + l = len(m.Payload) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + if m.Occured != 0 { + n += 1 + sovMessage(uint64(m.Occured)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovMessage(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozMessage(x uint64) (n int) { + return sovMessage(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Message) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Message: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Message: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Channel", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Channel = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Subtopic", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Subtopic = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Publisher", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Publisher = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Protocol", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Protocol = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Payload", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Payload = append(m.Payload[:0], dAtA[iNdEx:postIndex]...) + if m.Payload == nil { + m.Payload = []byte{} + } + iNdEx = postIndex + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Occured", wireType) + } + m.Occured = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Occured |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipMessage(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthMessage + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupMessage + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthMessage + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthMessage = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMessage = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupMessage = fmt.Errorf("proto: unexpected end of group") +) diff --git a/message.proto b/message.proto new file mode 100644 index 0000000000..07ec5a38e1 --- /dev/null +++ b/message.proto @@ -0,0 +1,15 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; +package mainflux; + +// Message represents a message emitted by the Mainflux adapters layer. +message Message { + string channel = 1; + string subtopic = 2; + string publisher = 3; + string protocol = 4; + bytes payload = 5; + int64 occured = 6; +} diff --git a/mqtt/adapter.go b/mqtt/adapter.go index 2d14bf9e0d..55eaba698a 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -12,9 +12,7 @@ import ( "strings" "time" - "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/mqtt/redis" "github.com/mainflux/mproxy/pkg/session" @@ -39,38 +37,38 @@ var ( // Event implements events.Event interface type handler struct { - broker broker.Nats - tc mainflux.ThingsServiceClient - tracer opentracing.Tracer - logger logger.Logger - es redis.EventStore + publishers []mainflux.Publisher + tc mainflux.ThingsServiceClient + tracer opentracing.Tracer + logger logger.Logger + es redis.EventStore } // New creates new Event entity -func New(broker broker.Nats, tc mainflux.ThingsServiceClient, es redis.EventStore, +func New(publishers []mainflux.Publisher, tc mainflux.ThingsServiceClient, es redis.EventStore, logger logger.Logger, tracer opentracing.Tracer) session.Handler { return &handler{ - broker: broker, - tc: tc, - es: es, - tracer: tracer, - logger: logger, + publishers: publishers, + tc: tc, + es: es, + tracer: tracer, + logger: logger, } } // AuthConnect is called on device connection, // prior forwarding to the MQTT broker -func (e *handler) AuthConnect(c *session.Client) error { +func (h *handler) AuthConnect(c *session.Client) error { if c == nil { return errInvalidConnect } - e.logger.Info(fmt.Sprintf("AuthConnect - client ID: %s, username: %s", c.ID, c.Username)) + h.logger.Info(fmt.Sprintf("AuthConnect - client ID: %s, username: %s", c.ID, c.Username)) t := &mainflux.Token{ Value: string(c.Password), } - thid, err := e.tc.Identify(context.TODO(), t) + thid, err := h.tc.Identify(context.TODO(), t) if err != nil { return err } @@ -79,8 +77,8 @@ func (e *handler) AuthConnect(c *session.Client) error { return errUnauthorizedAccess } - if err := e.es.Connect(c.Username); err != nil { - e.logger.Warn("Failed to publish connect event: " + err.Error()) + if err := h.es.Connect(c.Username); err != nil { + h.logger.Warn("Failed to publish connect event: " + err.Error()) } return nil @@ -88,30 +86,30 @@ func (e *handler) AuthConnect(c *session.Client) error { // AuthPublish is called on device publish, // prior forwarding to the MQTT broker -func (e *handler) AuthPublish(c *session.Client, topic *string, payload *[]byte) error { +func (h *handler) AuthPublish(c *session.Client, topic *string, payload *[]byte) error { if c == nil { return errNilClient } if topic == nil { return errNilTopicPub } - e.logger.Info("AuthPublish - client ID: " + c.ID + " topic: " + *topic) - return e.authAccess(c.Username, *topic) + h.logger.Info("AuthPublish - client ID: " + c.ID + " topic: " + *topic) + return h.authAccess(c.Username, *topic) } // AuthSubscribe is called on device publish, // prior forwarding to the MQTT broker -func (e *handler) AuthSubscribe(c *session.Client, topics *[]string) error { +func (h *handler) AuthSubscribe(c *session.Client, topics *[]string) error { if c == nil { return errNilClient } if topics == nil || *topics == nil { return errNilTopicSub } - e.logger.Info("AuthSubscribe - client ID: " + c.ID + " topics: " + strings.Join(*topics, ",")) + h.logger.Info("AuthSubscribe - client ID: " + c.ID + " topics: " + strings.Join(*topics, ",")) for _, v := range *topics { - if err := e.authAccess(c.Username, v); err != nil { + if err := h.authAccess(c.Username, v); err != nil { return err } @@ -121,27 +119,27 @@ func (e *handler) AuthSubscribe(c *session.Client, topics *[]string) error { } // Connect - after client sucesfully connected -func (e *handler) Connect(c *session.Client) { +func (h *handler) Connect(c *session.Client) { if c == nil { - e.logger.Error("Nil client connect") + h.logger.Error("Nil client connect") return } - e.logger.Info("Connect - client with ID: " + c.ID) + h.logger.Info("Connect - client with ID: " + c.ID) } // Publish - after client sucesfully published -func (e *handler) Publish(c *session.Client, topic *string, payload *[]byte) { +func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { if c == nil { - e.logger.Error("Nil client publish") + h.logger.Error("Nil client publish") return } - e.logger.Info("Publish - client ID " + c.ID + " to the topic: " + *topic) + h.logger.Info("Publish - client ID " + c.ID + " to the topic: " + *topic) // Topics are in the format: // channels//messages//.../ct/ channelParts := channelRegExp.FindStringSubmatch(*topic) if len(channelParts) < 1 { - e.logger.Info("Error in mqtt publish %s" + errMalformedData.Error()) + h.logger.Info("Error in mqtt publish %s" + errMalformedData.Error()) return } @@ -150,65 +148,61 @@ func (e *handler) Publish(c *session.Client, topic *string, payload *[]byte) { subtopic, err := parseSubtopic(subtopic) if err != nil { - e.logger.Info("Error in mqtt publish: " + err.Error()) + h.logger.Info("Error in mqtt publish: " + err.Error()) return } - created, err := ptypes.TimestampProto(time.Now()) - if err != nil { - e.logger.Info("Error in mqtt publish: " + err.Error()) - return - } - - msg := broker.Message{ + msg := mainflux.Message{ Protocol: protocol, Channel: chanID, Subtopic: subtopic, Publisher: c.Username, Payload: *payload, - Created: created, + Occured: time.Now().UnixNano(), } - if err := e.broker.Publish(context.TODO(), "", msg); err != nil { - e.logger.Info("Error publishing to Mainflux " + err.Error()) + for _, pub := range h.publishers { + if err := pub.Publish(msg); err != nil { + h.logger.Info("Error publishing to Mainflux " + err.Error()) + } } } // Subscribe - after client sucesfully subscribed -func (e *handler) Subscribe(c *session.Client, topics *[]string) { +func (h *handler) Subscribe(c *session.Client, topics *[]string) { if c == nil { - e.logger.Error("Nil client subscribe") + h.logger.Error("Nil client subscribe") return } - e.logger.Info("Subscribe - client ID: " + c.ID + ", to topics: " + strings.Join(*topics, ",")) + h.logger.Info("Subscribe - client ID: " + c.ID + ", to topics: " + strings.Join(*topics, ",")) } // Unsubscribe - after client unsubscribed -func (e *handler) Unsubscribe(c *session.Client, topics *[]string) { +func (h *handler) Unsubscribe(c *session.Client, topics *[]string) { if c == nil { - e.logger.Error("Nil client unsubscribe") + h.logger.Error("Nil client unsubscribe") return } - e.logger.Info("Unubscribe - client ID: " + c.ID + ", form topics: " + strings.Join(*topics, ",")) + h.logger.Info("Unubscribe - client ID: " + c.ID + ", form topics: " + strings.Join(*topics, ",")) } // Disconnect - connection with broker or client lost -func (e *handler) Disconnect(c *session.Client) { +func (h *handler) Disconnect(c *session.Client) { if c == nil { - e.logger.Error("Nil client disconnect") + h.logger.Error("Nil client disconnect") return } - e.logger.Info("Disconnect - Client with ID: " + c.ID + " and username " + c.Username + " disconnected") - if err := e.es.Disconnect(c.Username); err != nil { - e.logger.Warn("Failed to publish disconnect event: " + err.Error()) + h.logger.Info("Disconnect - Client with ID: " + c.ID + " and username " + c.Username + " disconnected") + if err := h.es.Disconnect(c.Username); err != nil { + h.logger.Warn("Failed to publish disconnect event: " + err.Error()) } } -func (e *handler) authAccess(username string, topic string) error { +func (h *handler) authAccess(username string, topic string) error { // Topics are in the format: // channels//messages//.../ct/ if !channelRegExp.Match([]byte(topic)) { - e.logger.Info("Malformed topic: " + topic) + h.logger.Info("Malformed topic: " + topic) return errMalformedTopic } @@ -223,12 +217,8 @@ func (e *handler) authAccess(username string, topic string) error { ThingID: username, ChanID: chanID, } - _, err := e.tc.CanAccessByID(context.TODO(), ar) - if err != nil { - return err - } - - return nil + _, err := h.tc.CanAccessByID(context.TODO(), ar) + return err } func parseSubtopic(subtopic string) (string, error) { diff --git a/nats/pubsub.go b/nats/pubsub.go new file mode 100644 index 0000000000..403d542099 --- /dev/null +++ b/nats/pubsub.go @@ -0,0 +1,110 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package nats + +import ( + "errors" + "fmt" + "sync" + + "github.com/gogo/protobuf/proto" + "github.com/mainflux/mainflux" + log "github.com/mainflux/mainflux/logger" + broker "github.com/nats-io/nats.go" +) + +const chansPrefix = "channels" + +// SubjectAllChannels represents subject to subscribe for all the channels. +const SubjectAllChannels = "channels.>" + +var errNotSubscribed = errors.New("not subscribed") + +var _ mainflux.PubSub = (*nats)(nil) + +type nats struct { + conn *broker.Conn + subscription *broker.Subscription + logger log.Logger + mu sync.Mutex + subject string + queue string +} + +// New returns NATS message broker. +func New(conn *broker.Conn, subject, queue string, logger log.Logger) mainflux.PubSub { + return &nats{ + conn: conn, + queue: queue, + logger: logger, + subject: subject, + } +} + +func (n *nats) Publish(msg mainflux.Message) error { + data, err := proto.Marshal(&msg) + if err != nil { + return err + } + + subject := fmt.Sprintf("%s.%s", chansPrefix, msg.Channel) + if msg.Subtopic != "" { + subject = fmt.Sprintf("%s.%s", subject, msg.Subtopic) + } + if err := n.conn.Publish(subject, data); err != nil { + return err + } + + return nil +} + +func (n *nats) Subscribe(subHandler mainflux.SubscribeHandler) error { + n.mu.Lock() + defer n.mu.Unlock() + + sub, err := n.subscribe(subHandler) + if err != nil { + return err + } + n.subscription = sub + return nil +} + +func (n *nats) Unsubscribe() error { + n.mu.Lock() + defer n.mu.Unlock() + + if n.subscription == nil { + return errNotSubscribed + } + if err := n.subscription.Unsubscribe(); err != nil { + return err + } + n.subscription = nil + return nil +} + +func (n *nats) subscribe(subHandler mainflux.SubscribeHandler) (*broker.Subscription, error) { + if n.queue != "" { + return n.conn.QueueSubscribe(n.subject, n.queue, n.natsHandler(subHandler)) + } + ps := SubjectAllChannels + if n.subject != "" { + ps = fmt.Sprintf("%s.%s", chansPrefix, n.subject) + } + return n.conn.Subscribe(ps, n.natsHandler(subHandler)) +} + +func (n *nats) natsHandler(h mainflux.SubscribeHandler) broker.MsgHandler { + return func(m *broker.Msg) { + var msg mainflux.Message + if err := proto.Unmarshal(m.Data, &msg); err != nil { + n.logger.Warn(fmt.Sprintf("Failed to unmarshal received message: %s", err)) + return + } + if err := h(msg); err != nil { + n.logger.Warn(fmt.Sprintf("Failed handle Mainflux message: %s", err)) + } + } +} diff --git a/pubsub.go b/pubsub.go new file mode 100644 index 0000000000..8f5bcdec06 --- /dev/null +++ b/pubsub.go @@ -0,0 +1,26 @@ +package mainflux + +// Publisher specifies message publishing API. +type Publisher interface { + // Publishes message to the stream. + Publish(msg Message) error +} + +// SubscribeHandler represents Message handler for Subscriber. +type SubscribeHandler func(Message) error + +// Subscriber specifies message subscription API. +type Subscriber interface { + // Subscribe subscribes to the message stream and consumes messages. + Subscribe(SubscribeHandler) error + + // Unsubscribe unsubscribes from the message stream and + // stops consuming messages. + Unsubscribe() error +} + +// PubSub represents aggregation interface for publisher and subscriber. +type PubSub interface { + Publisher + Subscriber +} diff --git a/transformers/senml/transformer.go b/transformers/senml/transformer.go index 7e729da5f7..a29351f3f7 100644 --- a/transformers/senml/transformer.go +++ b/transformers/senml/transformer.go @@ -4,7 +4,9 @@ package senml import ( - "github.com/mainflux/mainflux/broker" + "time" + + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/transformers" "github.com/mainflux/senml" @@ -43,7 +45,7 @@ func New(contentFormat string) transformers.Transformer { } } -func (t transformer) Transform(msg broker.Message) (interface{}, error) { +func (t transformer) Transform(msg mainflux.Message) (interface{}, error) { raw, err := senml.Decode(msg.Payload, t.format) if err != nil { return nil, errors.Wrap(errDecode, err) @@ -57,10 +59,9 @@ func (t transformer) Transform(msg broker.Message) (interface{}, error) { msgs := make([]Message, len(normalized.Records)) for i, v := range normalized.Records { // Use reception timestamp if SenML messsage Time is missing - time := v.Time - if time == 0 { - // Convert the timestamp into float64 with nanoseconds precision - time = float64(msg.Created.GetSeconds()) + float64(msg.Created.GetNanos())/float64(1e9) + t := v.Time + if t == 0 { + t = float64(time.Now().UnixNano()) } msgs[i] = Message{ @@ -70,7 +71,7 @@ func (t transformer) Transform(msg broker.Message) (interface{}, error) { Protocol: msg.Protocol, Name: v.Name, Unit: v.Unit, - Time: time, + Time: t, UpdateTime: v.UpdateTime, Value: v.Value, BoolValue: v.BoolValue, diff --git a/transformers/senml/transformer_test.go b/transformers/senml/transformer_test.go index 1ccd368e95..73ac1f8993 100644 --- a/transformers/senml/transformer_test.go +++ b/transformers/senml/transformer_test.go @@ -8,7 +8,7 @@ import ( "fmt" "testing" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/transformers/senml" mfsenml "github.com/mainflux/senml" @@ -24,7 +24,7 @@ func TestTransformJSON(t *testing.T) { require.Nil(t, err, "Decoding JSON expected to succeed") tr := senml.New(senml.JSON) - msg := broker.Message{ + msg := mainflux.Message{ Channel: "channel", Subtopic: "subtopic", Publisher: "publisher", @@ -55,7 +55,7 @@ func TestTransformJSON(t *testing.T) { cases := []struct { desc string - msg broker.Message + msg mainflux.Message msgs interface{} err error }{ @@ -91,7 +91,7 @@ func TestTransformCBOR(t *testing.T) { require.Nil(t, err, "Decoding CBOR expected to succeed") tr := senml.New(senml.CBOR) - msg := broker.Message{ + msg := mainflux.Message{ Channel: "channel", Subtopic: "subtopic", Publisher: "publisher", @@ -109,23 +109,24 @@ func TestTransformCBOR(t *testing.T) { val := 52.0 sum := 110.0 - msgs := []senml.Message{senml.Message{ - Channel: "channel", - Subtopic: "subtopic", - Publisher: "publisher", - Protocol: "protocol", - Name: "base-namename", - Unit: "unit", - Time: 400, - UpdateTime: 150, - Value: &val, - Sum: &sum, - }, + msgs := []senml.Message{ + { + Channel: "channel", + Subtopic: "subtopic", + Publisher: "publisher", + Protocol: "protocol", + Name: "base-namename", + Unit: "unit", + Time: 400, + UpdateTime: 150, + Value: &val, + Sum: &sum, + }, } cases := []struct { desc string - msg broker.Message + msg mainflux.Message msgs interface{} err error }{ diff --git a/transformers/service.go b/transformers/service.go index 8d9f792efe..ec0cb9a0ba 100644 --- a/transformers/service.go +++ b/transformers/service.go @@ -3,10 +3,10 @@ package transformers -import "github.com/mainflux/mainflux/broker" +import "github.com/mainflux/mainflux" // Transformer specifies API form Message transformer. type Transformer interface { // Transform Mainflux message to any other format. - Transform(msg broker.Message) (interface{}, error) + Transform(msg mainflux.Message) (interface{}, error) } diff --git a/writers/writer.go b/writers/writer.go index c2062375ff..8bb084a45a 100644 --- a/writers/writer.go +++ b/writers/writer.go @@ -8,22 +8,22 @@ import ( "io/ioutil" "github.com/BurntSushi/toml" - "github.com/gogo/protobuf/proto" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/transformers" "github.com/mainflux/mainflux/transformers/senml" - "github.com/nats-io/nats.go" + broker "github.com/nats-io/nats.go" ) var ( - errOpenConfFile = errors.New("Unable to open configuration file") - errParseConfFile = errors.New("Unable to parse configuration file") + errOpenConfFile = errors.New("unable to open configuration file") + errParseConfFile = errors.New("unable to parse configuration file") + errMessageConversion = errors.New("error conversing transformed messages") ) type consumer struct { - broker broker.Nats repo MessageRepository transformer transformers.Transformer logger logger.Logger @@ -32,9 +32,8 @@ type consumer struct { // Start method starts consuming messages received from NATS. // This method transforms messages to SenML format before // using MessageRepository to store them. -func Start(broker broker.Nats, repo MessageRepository, transformer transformers.Transformer, queue string, subjectsCfgPath string, logger logger.Logger) error { +func Start(conn *broker.Conn, repo MessageRepository, transformer transformers.Transformer, queue string, subjectsCfgPath string, logger logger.Logger) error { c := consumer{ - broker: broker, repo: repo, transformer: transformer, logger: logger, @@ -46,36 +45,25 @@ func Start(broker broker.Nats, repo MessageRepository, transformer transformers. } for _, subject := range subjects { - _, err := broker.QueueSubscribe(subject, queue, c.consume) - if err != nil { + sub := nats.New(conn, subject, queue, logger) + if err := sub.Subscribe(c.handler); err != nil { return err } } return nil } -func (c *consumer) consume(m *nats.Msg) { - var msg broker.Message - if err := proto.Unmarshal(m.Data, &msg); err != nil { - c.logger.Warn(fmt.Sprintf("Failed to unmarshal received message: %s", err)) - return - } - +func (c *consumer) handler(msg mainflux.Message) error { t, err := c.transformer.Transform(msg) if err != nil { - c.logger.Warn(fmt.Sprintf("Failed to tranform received message: %s", err)) - return + return err } msgs, ok := t.([]senml.Message) if !ok { - c.logger.Warn("Invalid message format from the Transformer output.") - return + return errMessageConversion } - if err := c.repo.Save(msgs...); err != nil { - c.logger.Warn(fmt.Sprintf("Failed to save message: %s", err)) - return - } + return c.repo.Save(msgs...) } type filterConfig struct { @@ -89,12 +77,12 @@ type subjectsConfig struct { func loadSubjectsConfig(subjectsConfigPath string) ([]string, error) { data, err := ioutil.ReadFile(subjectsConfigPath) if err != nil { - return []string{broker.SubjectAllChannels}, errors.Wrap(errOpenConfFile, err) + return []string{nats.SubjectAllChannels}, errors.Wrap(errOpenConfFile, err) } var subjectsCfg subjectsConfig if err := toml.Unmarshal(data, &subjectsCfg); err != nil { - return []string{broker.SubjectAllChannels}, errors.Wrap(errParseConfFile, err) + return []string{nats.SubjectAllChannels}, errors.Wrap(errParseConfFile, err) } return subjectsCfg.Subjects.List, nil From 20e619d632b44aa608699a1bc578961544634f74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Thu, 23 Apr 2020 18:02:25 +0200 Subject: [PATCH 02/28] Rename SubscribeHandler to MessageHandler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- nats/pubsub.go | 6 +++--- pubsub.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nats/pubsub.go b/nats/pubsub.go index 403d542099..76e8c8a665 100644 --- a/nats/pubsub.go +++ b/nats/pubsub.go @@ -59,7 +59,7 @@ func (n *nats) Publish(msg mainflux.Message) error { return nil } -func (n *nats) Subscribe(subHandler mainflux.SubscribeHandler) error { +func (n *nats) Subscribe(subHandler mainflux.MessageHandler) error { n.mu.Lock() defer n.mu.Unlock() @@ -85,7 +85,7 @@ func (n *nats) Unsubscribe() error { return nil } -func (n *nats) subscribe(subHandler mainflux.SubscribeHandler) (*broker.Subscription, error) { +func (n *nats) subscribe(subHandler mainflux.MessageHandler) (*broker.Subscription, error) { if n.queue != "" { return n.conn.QueueSubscribe(n.subject, n.queue, n.natsHandler(subHandler)) } @@ -96,7 +96,7 @@ func (n *nats) subscribe(subHandler mainflux.SubscribeHandler) (*broker.Subscrip return n.conn.Subscribe(ps, n.natsHandler(subHandler)) } -func (n *nats) natsHandler(h mainflux.SubscribeHandler) broker.MsgHandler { +func (n *nats) natsHandler(h mainflux.MessageHandler) broker.MsgHandler { return func(m *broker.Msg) { var msg mainflux.Message if err := proto.Unmarshal(m.Data, &msg); err != nil { diff --git a/pubsub.go b/pubsub.go index 8f5bcdec06..51e47a2797 100644 --- a/pubsub.go +++ b/pubsub.go @@ -6,13 +6,13 @@ type Publisher interface { Publish(msg Message) error } -// SubscribeHandler represents Message handler for Subscriber. -type SubscribeHandler func(Message) error +// MessageHandler represents Message handler for Subscriber. +type MessageHandler func(Message) error // Subscriber specifies message subscription API. type Subscriber interface { // Subscribe subscribes to the message stream and consumes messages. - Subscribe(SubscribeHandler) error + Subscribe(MessageHandler) error // Unsubscribe unsubscribes from the message stream and // stops consuming messages. From 511f620c7b49de5bd7e8861ecda7722d8c9175bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Sun, 26 Apr 2020 14:37:47 +0200 Subject: [PATCH 03/28] Remove `Auth` event logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- mqtt/adapter.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mqtt/adapter.go b/mqtt/adapter.go index 55eaba698a..6f7e3a2751 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -6,7 +6,6 @@ package mqtt import ( "context" "errors" - "fmt" "net/url" "regexp" "strings" @@ -62,7 +61,6 @@ func (h *handler) AuthConnect(c *session.Client) error { if c == nil { return errInvalidConnect } - h.logger.Info(fmt.Sprintf("AuthConnect - client ID: %s, username: %s", c.ID, c.Username)) t := &mainflux.Token{ Value: string(c.Password), @@ -93,7 +91,7 @@ func (h *handler) AuthPublish(c *session.Client, topic *string, payload *[]byte) if topic == nil { return errNilTopicPub } - h.logger.Info("AuthPublish - client ID: " + c.ID + " topic: " + *topic) + return h.authAccess(c.Username, *topic) } @@ -106,7 +104,6 @@ func (h *handler) AuthSubscribe(c *session.Client, topics *[]string) error { if topics == nil || *topics == nil { return errNilTopicSub } - h.logger.Info("AuthSubscribe - client ID: " + c.ID + " topics: " + strings.Join(*topics, ",")) for _, v := range *topics { if err := h.authAccess(c.Username, v); err != nil { From f64bce712ffd912d41787f43b97dfab418f7e707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Sun, 26 Apr 2020 21:58:47 +0200 Subject: [PATCH 04/28] Update message pubsub APi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 5 ++- cmd/http/main.go | 5 +-- cmd/influxdb-writer/main.go | 5 ++- cmd/mongodb-writer/main.go | 5 ++- cmd/postgres-writer/main.go | 5 ++- http/mocks/publisher.go | 2 +- nats/pubsub.go | 82 ++++++++++++++++++++++-------------- pubsub.go | 8 ++-- writers/writer.go | 6 +-- 9 files changed, 75 insertions(+), 48 deletions(-) diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 7daed0b5ea..386e9ac33e 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -17,6 +17,7 @@ import ( "github.com/gocql/gocql" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -76,12 +77,14 @@ func main() { } defer nc.Close() + n := nats.New(nc, "", logger) + session := connectToCassandra(cfg.dbCfg, logger) defer session.Close() repo := newService(session, logger) st := senml.New(cfg.contentType) - if err := writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Cassandra writer: %s", err)) } diff --git a/cmd/http/main.go b/cmd/http/main.go index 96820ba65a..7eab69db97 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -63,7 +63,6 @@ type config struct { } func main() { - cfg := loadConfig() logger, err := logger.New(os.Stdout, cfg.logLevel) @@ -87,10 +86,10 @@ func main() { } defer nc.Close() - b := nats.New(nc, "", "", logger) + n := nats.New(nc, "", logger) tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - svc := adapter.New(b, tc) + svc := adapter.New(n, tc) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index 3e6be292a5..02a74afa1d 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -15,6 +15,7 @@ import ( influxdata "github.com/influxdata/influxdb/client/v2" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -77,6 +78,8 @@ func main() { } defer nc.Close() + n := nats.New(nc, "", logger) + client, err := influxdata.NewHTTPClient(clientCfg) if err != nil { logger.Error(fmt.Sprintf("Failed to create InfluxDB client: %s", err)) @@ -91,7 +94,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start InfluxDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index d19fe3cfe7..9750f71dc2 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -15,6 +15,7 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -73,6 +74,8 @@ func main() { } defer nc.Close() + n := nats.New(nc, "", logger) + addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) if err != nil { @@ -88,7 +91,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start MongoDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index 694988dc87..f17a3e315a 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -15,6 +15,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -82,12 +83,14 @@ func main() { } defer nc.Close() + n := nats.New(nc, "", logger) + db := connectToDB(cfg.dbConfig, logger) defer db.Close() repo := newService(db, logger) st := senml.New(cfg.contentType) - if err = writers.Start(nc, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err = writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) } diff --git a/http/mocks/publisher.go b/http/mocks/publisher.go index 86fe3f836d..7d08dac016 100644 --- a/http/mocks/publisher.go +++ b/http/mocks/publisher.go @@ -14,6 +14,6 @@ func NewPublisher() mainflux.Publisher { return mockPublisher{} } -func (pub mockPublisher) Publish(mainflux.Message) error { +func (pub mockPublisher) Publish(topic string, msg mainflux.Message) error { return nil } diff --git a/nats/pubsub.go b/nats/pubsub.go index 76e8c8a665..f772c5c2b7 100644 --- a/nats/pubsub.go +++ b/nats/pubsub.go @@ -19,36 +19,37 @@ const chansPrefix = "channels" // SubjectAllChannels represents subject to subscribe for all the channels. const SubjectAllChannels = "channels.>" -var errNotSubscribed = errors.New("not subscribed") +var ( + errAlreadySubscribed = errors.New("already subscribed to topic") + errNotSubscribed = errors.New("not subscribed") +) var _ mainflux.PubSub = (*nats)(nil) type nats struct { - conn *broker.Conn - subscription *broker.Subscription - logger log.Logger - mu sync.Mutex - subject string - queue string + conn *broker.Conn + subscriptions map[string]*broker.Subscription + logger log.Logger + mu sync.Mutex + queue string } // New returns NATS message broker. -func New(conn *broker.Conn, subject, queue string, logger log.Logger) mainflux.PubSub { +func New(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { return &nats{ - conn: conn, - queue: queue, - logger: logger, - subject: subject, + conn: conn, + queue: queue, + logger: logger, } } -func (n *nats) Publish(msg mainflux.Message) error { +func (n *nats) Publish(topic string, msg mainflux.Message) error { data, err := proto.Marshal(&msg) if err != nil { return err } - subject := fmt.Sprintf("%s.%s", chansPrefix, msg.Channel) + subject := fmt.Sprintf("%s.%s", chansPrefix, topic) if msg.Subtopic != "" { subject = fmt.Sprintf("%s.%s", subject, msg.Subtopic) } @@ -59,42 +60,59 @@ func (n *nats) Publish(msg mainflux.Message) error { return nil } -func (n *nats) Subscribe(subHandler mainflux.MessageHandler) error { +func (n *nats) Subscribe(topic string, handler mainflux.MessageHandler) error { n.mu.Lock() defer n.mu.Unlock() - - sub, err := n.subscribe(subHandler) + if _, ok := n.subscriptions[topic]; ok { + return errAlreadySubscribed + } + s := SubjectAllChannels + if topic != "" { + s = fmt.Sprintf("%s.%s", chansPrefix, topic) + } + if n.queue != "" { + sub, err := n.conn.QueueSubscribe(s, n.queue, n.natsHandler(handler)) + if err != nil { + return err + } + n.subscriptions[s] = sub + return nil + } + sub, err := n.conn.Subscribe(s, n.natsHandler(handler)) if err != nil { return err } - n.subscription = sub + n.subscriptions[s] = sub return nil } -func (n *nats) Unsubscribe() error { +func (n *nats) Unsubscribe(topic string) error { n.mu.Lock() defer n.mu.Unlock() - if n.subscription == nil { + sub, ok := n.subscriptions[topic] + if !ok { return errNotSubscribed } - if err := n.subscription.Unsubscribe(); err != nil { + + if err := sub.Unsubscribe(); err != nil { return err } - n.subscription = nil + + delete(n.subscriptions, topic) return nil } -func (n *nats) subscribe(subHandler mainflux.MessageHandler) (*broker.Subscription, error) { - if n.queue != "" { - return n.conn.QueueSubscribe(n.subject, n.queue, n.natsHandler(subHandler)) - } - ps := SubjectAllChannels - if n.subject != "" { - ps = fmt.Sprintf("%s.%s", chansPrefix, n.subject) - } - return n.conn.Subscribe(ps, n.natsHandler(subHandler)) -} +// func (n *nats) subscribe(topic string, handler mainflux.MessageHandler) (*broker.Subscription, error) { +// if n.queue != "" { +// return n.conn.QueueSubscribe(topic, n.queue, n.natsHandler(handler)) +// } +// ps := SubjectAllChannels +// if topic != "" { +// ps = fmt.Sprintf("%s.%s", chansPrefix, topic) +// } +// return n.conn.Subscribe(ps, n.natsHandler(handler)) +// } func (n *nats) natsHandler(h mainflux.MessageHandler) broker.MsgHandler { return func(m *broker.Msg) { diff --git a/pubsub.go b/pubsub.go index 51e47a2797..4f41b4adc2 100644 --- a/pubsub.go +++ b/pubsub.go @@ -3,20 +3,20 @@ package mainflux // Publisher specifies message publishing API. type Publisher interface { // Publishes message to the stream. - Publish(msg Message) error + Publish(topic string, msg Message) error } // MessageHandler represents Message handler for Subscriber. -type MessageHandler func(Message) error +type MessageHandler func(msg Message) error // Subscriber specifies message subscription API. type Subscriber interface { // Subscribe subscribes to the message stream and consumes messages. - Subscribe(MessageHandler) error + Subscribe(topic string, handler MessageHandler) error // Unsubscribe unsubscribes from the message stream and // stops consuming messages. - Unsubscribe() error + Unsubscribe(topic string) error } // PubSub represents aggregation interface for publisher and subscriber. diff --git a/writers/writer.go b/writers/writer.go index 8bb084a45a..2a50886e2a 100644 --- a/writers/writer.go +++ b/writers/writer.go @@ -14,7 +14,6 @@ import ( "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/transformers" "github.com/mainflux/mainflux/transformers/senml" - broker "github.com/nats-io/nats.go" ) var ( @@ -32,7 +31,7 @@ type consumer struct { // Start method starts consuming messages received from NATS. // This method transforms messages to SenML format before // using MessageRepository to store them. -func Start(conn *broker.Conn, repo MessageRepository, transformer transformers.Transformer, queue string, subjectsCfgPath string, logger logger.Logger) error { +func Start(sub mainflux.Subscriber, repo MessageRepository, transformer transformers.Transformer, queue string, subjectsCfgPath string, logger logger.Logger) error { c := consumer{ repo: repo, transformer: transformer, @@ -45,8 +44,7 @@ func Start(conn *broker.Conn, repo MessageRepository, transformer transformers.T } for _, subject := range subjects { - sub := nats.New(conn, subject, queue, logger) - if err := sub.Subscribe(c.handler); err != nil { + if err := sub.Subscribe(subject, c.handler); err != nil { return err } } From ecd5f73efdd2fdfd143b11ece7c9ff79445b27b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Sun, 26 Apr 2020 23:27:13 +0200 Subject: [PATCH 05/28] Fix topics handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- nats/pubsub.go | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/nats/pubsub.go b/nats/pubsub.go index f772c5c2b7..15e1cfe87c 100644 --- a/nats/pubsub.go +++ b/nats/pubsub.go @@ -22,24 +22,26 @@ const SubjectAllChannels = "channels.>" var ( errAlreadySubscribed = errors.New("already subscribed to topic") errNotSubscribed = errors.New("not subscribed") + errEmptyTopic = errors.New("empty topic") ) var _ mainflux.PubSub = (*nats)(nil) type nats struct { conn *broker.Conn - subscriptions map[string]*broker.Subscription logger log.Logger mu sync.Mutex queue string + subscriptions map[string]*broker.Subscription } // New returns NATS message broker. func New(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { return &nats{ - conn: conn, - queue: queue, - logger: logger, + conn: conn, + queue: queue, + logger: logger, + subscriptions: make(map[string]*broker.Subscription), } } @@ -61,35 +63,40 @@ func (n *nats) Publish(topic string, msg mainflux.Message) error { } func (n *nats) Subscribe(topic string, handler mainflux.MessageHandler) error { + if topic == "" { + return errEmptyTopic + } n.mu.Lock() defer n.mu.Unlock() if _, ok := n.subscriptions[topic]; ok { return errAlreadySubscribed } - s := SubjectAllChannels - if topic != "" { - s = fmt.Sprintf("%s.%s", chansPrefix, topic) - } + topic = fmt.Sprintf("%s.%s", chansPrefix, topic) if n.queue != "" { - sub, err := n.conn.QueueSubscribe(s, n.queue, n.natsHandler(handler)) + sub, err := n.conn.QueueSubscribe(topic, n.queue, n.natsHandler(handler)) if err != nil { return err } - n.subscriptions[s] = sub + n.subscriptions[topic] = sub return nil } - sub, err := n.conn.Subscribe(s, n.natsHandler(handler)) + sub, err := n.conn.Subscribe(topic, n.natsHandler(handler)) if err != nil { return err } - n.subscriptions[s] = sub + n.subscriptions[topic] = sub return nil } func (n *nats) Unsubscribe(topic string) error { + if topic == "" { + return errEmptyTopic + } n.mu.Lock() defer n.mu.Unlock() + topic = fmt.Sprintf("%s.%s", chansPrefix, topic) + sub, ok := n.subscriptions[topic] if !ok { return errNotSubscribed @@ -103,17 +110,6 @@ func (n *nats) Unsubscribe(topic string) error { return nil } -// func (n *nats) subscribe(topic string, handler mainflux.MessageHandler) (*broker.Subscription, error) { -// if n.queue != "" { -// return n.conn.QueueSubscribe(topic, n.queue, n.natsHandler(handler)) -// } -// ps := SubjectAllChannels -// if topic != "" { -// ps = fmt.Sprintf("%s.%s", chansPrefix, topic) -// } -// return n.conn.Subscribe(ps, n.natsHandler(handler)) -// } - func (n *nats) natsHandler(h mainflux.MessageHandler) broker.MsgHandler { return func(m *broker.Msg) { var msg mainflux.Message From 62af5aca1aa7be8c488704e933b1ba960d83994f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:09:48 +0200 Subject: [PATCH 06/28] Update CoAP adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/coap/main.go | 13 ++++++++----- coap/adapter.go | 32 +++++++++++--------------------- coap/api/logging.go | 7 +++---- coap/api/metrics.go | 7 +++---- coap/api/transport.go | 13 +++---------- coap/observer.go | 6 +++--- 6 files changed, 31 insertions(+), 47 deletions(-) diff --git a/cmd/coap/main.go b/cmd/coap/main.go index afe6ba2560..94b9ca731f 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -18,11 +18,12 @@ import ( gocoap "github.com/dustin/go-coap" kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/coap" "github.com/mainflux/mainflux/coap/api" logger "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" + broker "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -81,14 +82,16 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) respChan := make(chan string, 10000) - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() - svc := coap.New(b, logger, cc, respChan) + n := nats.New(nc, "", logger) + + svc := coap.New(n, logger, cc, respChan) svc = api.LoggingMiddleware(svc, logger) diff --git a/coap/adapter.go b/coap/adapter.go index 0b622e0809..b97e980ce8 100644 --- a/coap/adapter.go +++ b/coap/adapter.go @@ -7,16 +7,12 @@ package coap import ( - "context" "fmt" "sync" "time" - "github.com/gogo/protobuf/proto" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" - "github.com/nats-io/nats.go" ) const ( @@ -34,7 +30,7 @@ const ( // Service specifies coap service API. type Service interface { // Publish Messssage - Publish(ctx context.Context, token string, msg broker.Message) error + Publish(msg mainflux.Message) error // Subscribes to channel with specified id, subtopic and adds subscription to // service map of subscriptions under given ID. @@ -48,17 +44,17 @@ var _ Service = (*adapterService)(nil) type adapterService struct { auth mainflux.ThingsServiceClient - broker broker.Nats + ps mainflux.PubSub log logger.Logger obs map[string]*Observer obsLock sync.Mutex } // New instantiates the CoAP adapter implementation. -func New(broker broker.Nats, log logger.Logger, auth mainflux.ThingsServiceClient, responses <-chan string) Service { +func New(ps mainflux.PubSub, log logger.Logger, auth mainflux.ThingsServiceClient, responses <-chan string) Service { as := &adapterService{ auth: auth, - broker: broker, + ps: ps, log: log, obs: make(map[string]*Observer), obsLock: sync.Mutex{}, @@ -111,8 +107,8 @@ func (svc *adapterService) listenResponses(responses <-chan string) { } } -func (svc *adapterService) Publish(ctx context.Context, token string, msg broker.Message) error { - return svc.broker.Publish(ctx, token, msg) +func (svc *adapterService) Publish(msg mainflux.Message) error { + return svc.ps.Publish(msg.Channel, msg) } func (svc *adapterService) Subscribe(chanID, subtopic, obsID string, o *Observer) error { @@ -121,15 +117,9 @@ func (svc *adapterService) Subscribe(chanID, subtopic, obsID string, o *Observer subject = fmt.Sprintf("%s.%s", chanID, subtopic) } - sub, err := svc.broker.Subscribe(subject, func(msg *nats.Msg) { - if msg == nil { - return - } - var m broker.Message - if err := proto.Unmarshal(msg.Data, &m); err != nil { - return - } - o.Messages <- m + err := svc.ps.Subscribe(subject, func(msg mainflux.Message) error { + o.Messages <- msg + return nil }) if err != nil { return err @@ -137,8 +127,8 @@ func (svc *adapterService) Subscribe(chanID, subtopic, obsID string, o *Observer go func() { <-o.Cancel - if err := sub.Unsubscribe(); err != nil { - svc.log.Error(fmt.Sprintf("Failed to unsubscribe from %s.%s", chanID, subtopic)) + if err := svc.ps.Unsubscribe(subject); err != nil { + svc.log.Error(fmt.Sprintf("Failed to unsubscribe from %s.%s due to %s", chanID, subtopic, err)) } }() diff --git a/coap/api/logging.go b/coap/api/logging.go index 7b9378f929..a070cd26e9 100644 --- a/coap/api/logging.go +++ b/coap/api/logging.go @@ -6,11 +6,10 @@ package api import ( - "context" "fmt" "time" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/coap" log "github.com/mainflux/mainflux/logger" ) @@ -27,7 +26,7 @@ func LoggingMiddleware(svc coap.Service, logger log.Logger) coap.Service { return &loggingMiddleware{logger, svc} } -func (lm *loggingMiddleware) Publish(ctx context.Context, token string, msg broker.Message) (err error) { +func (lm *loggingMiddleware) Publish(msg mainflux.Message) (err error) { defer func(begin time.Time) { destChannel := msg.Channel if msg.Subtopic != "" { @@ -41,7 +40,7 @@ func (lm *loggingMiddleware) Publish(ctx context.Context, token string, msg brok lm.logger.Info(fmt.Sprintf("%s without errors.", message)) }(time.Now()) - return lm.svc.Publish(ctx, token, msg) + return lm.svc.Publish(msg) } func (lm *loggingMiddleware) Subscribe(chanID, subtopic, obsID string, o *coap.Observer) (err error) { diff --git a/coap/api/metrics.go b/coap/api/metrics.go index 6ce7d93acf..7b033e11f0 100644 --- a/coap/api/metrics.go +++ b/coap/api/metrics.go @@ -6,11 +6,10 @@ package api import ( - "context" "time" "github.com/go-kit/kit/metrics" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/coap" ) @@ -31,13 +30,13 @@ func MetricsMiddleware(svc coap.Service, counter metrics.Counter, latency metric } } -func (mm *metricsMiddleware) Publish(ctx context.Context, token string, msg broker.Message) error { +func (mm *metricsMiddleware) Publish(msg mainflux.Message) error { defer func(begin time.Time) { mm.counter.With("method", "publish").Add(1) mm.latency.With("method", "publish").Observe(time.Since(begin).Seconds()) }(time.Now()) - return mm.svc.Publish(ctx, token, msg) + return mm.svc.Publish(msg) } func (mm *metricsMiddleware) Subscribe(chanID, subtopic, clientID string, o *coap.Observer) error { diff --git a/coap/api/transport.go b/coap/api/transport.go index 7fbf6dff3f..ff242af7a8 100644 --- a/coap/api/transport.go +++ b/coap/api/transport.go @@ -18,9 +18,7 @@ import ( gocoap "github.com/dustin/go-coap" "github.com/go-zoo/bone" - "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/coap" log "github.com/mainflux/mainflux/logger" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -225,21 +223,16 @@ func receive(svc coap.Service, msg *gocoap.Message) *gocoap.Message { return res } - created, err := ptypes.TimestampProto(time.Now()) - if err != nil { - return nil - } - - m := broker.Message{ + m := mainflux.Message{ Channel: chanID, Subtopic: subtopic, Publisher: publisher, Protocol: protocol, Payload: msg.Payload, - Created: created, + Occured: time.Now().UnixNano(), } - if err := svc.Publish(context.Background(), "", m); err != nil { + if err := svc.Publish(m); err != nil { res.Code = gocoap.InternalServerError } diff --git a/coap/observer.go b/coap/observer.go index 5db60c17ca..ad076fe6a4 100644 --- a/coap/observer.go +++ b/coap/observer.go @@ -6,7 +6,7 @@ package coap import ( "sync" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" ) // Observer is used to handle CoAP subscription. @@ -25,7 +25,7 @@ type Observer struct { expiredLock, msgIDLock sync.Mutex // Messages is used to receive messages from NATS. - Messages chan broker.Message + Messages chan mainflux.Message // Cancel channel is used to cancel observing resource. // Cancel channel should not be used to send or receive any @@ -36,7 +36,7 @@ type Observer struct { // NewObserver instantiates a new Observer. func NewObserver() *Observer { return &Observer{ - Messages: make(chan broker.Message), + Messages: make(chan mainflux.Message), Cancel: make(chan bool), } } From 4d806f4a0dd8917e3f2e4cbe9bfb7f63ae9c3b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:10:52 +0200 Subject: [PATCH 07/28] Update Twins service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/twins/main.go | 33 +++++++++++++++------------------ twins/api/logging.go | 4 ++-- twins/api/metrics.go | 4 ++-- twins/mocks/messages.go | 26 ++++---------------------- twins/service.go | 20 ++++++++++---------- 5 files changed, 33 insertions(+), 54 deletions(-) diff --git a/cmd/twins/main.go b/cmd/twins/main.go index 462a0fd54d..b608515aeb 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -16,18 +16,17 @@ import ( "time" kitprometheus "github.com/go-kit/kit/metrics/prometheus" - "github.com/gogo/protobuf/proto" "github.com/mainflux/mainflux" authapi "github.com/mainflux/mainflux/authn/api/grpc" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" localusers "github.com/mainflux/mainflux/things/users" "github.com/mainflux/mainflux/twins" "github.com/mainflux/mainflux/twins/api" twapi "github.com/mainflux/mainflux/twins/api/http" twmongodb "github.com/mainflux/mainflux/twins/mongodb" "github.com/mainflux/mainflux/twins/uuid" - "github.com/nats-io/nats.go" + broker "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -115,12 +114,14 @@ func main() { dbTracer, dbCloser := initJaeger("twins_db", cfg.jaegerURL, logger) defer dbCloser.Close() - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() + + n := nats.New(nc, queue, logger) ncTracer, ncCloser := initJaeger("twins_nats", cfg.jaegerURL, logger) defer ncCloser.Close() @@ -128,7 +129,7 @@ func main() { tracer, closer := initJaeger("twins", cfg.jaegerURL, logger) defer closer.Close() - svc := newService(b, ncTracer, cfg.channelID, auth, dbTracer, db, logger) + svc := newService(n, ncTracer, cfg.channelID, auth, dbTracer, db, logger) errs := make(chan error, 2) @@ -237,13 +238,13 @@ func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { return conn } -func newService(b broker.Nats, ncTracer opentracing.Tracer, chanID string, users mainflux.AuthNServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, logger logger.Logger) twins.Service { +func newService(ps mainflux.PubSub, ncTracer opentracing.Tracer, chanID string, users mainflux.AuthNServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, logger logger.Logger) twins.Service { twinRepo := twmongodb.NewTwinRepository(db) stateRepo := twmongodb.NewStateRepository(db) idp := uuid.New() - svc := twins.New(b, users, twinRepo, stateRepo, idp, chanID, logger) + svc := twins.New(ps, users, twinRepo, stateRepo, idp, chanID, logger) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( svc, @@ -261,21 +262,17 @@ func newService(b broker.Nats, ncTracer opentracing.Tracer, chanID string, users }, []string{"method"}), ) - _, err := b.QueueSubscribe(broker.SubjectAllChannels, queue, func(m *nats.Msg) { - var msg broker.Message - if err := proto.Unmarshal(m.Data, &msg); err != nil { - logger.Warn(fmt.Sprintf("Unmarshalling failed: %s", err)) - return - } - + err := ps.Subscribe("channels.>", func(msg mainflux.Message) error { if msg.Channel == chanID { - return + return nil } if err := svc.SaveStates(&msg); err != nil { logger.Error(fmt.Sprintf("State save failed: %s", err)) - return + return err } + + return nil }) if err != nil { logger.Error(err.Error()) diff --git a/twins/api/logging.go b/twins/api/logging.go index 182f7caa16..1d070c861f 100644 --- a/twins/api/logging.go +++ b/twins/api/logging.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" log "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/twins" ) @@ -79,7 +79,7 @@ func (lm *loggingMiddleware) ListTwins(ctx context.Context, token string, offset return lm.svc.ListTwins(ctx, token, offset, limit, name, metadata) } -func (lm *loggingMiddleware) SaveStates(msg *broker.Message) (err error) { +func (lm *loggingMiddleware) SaveStates(msg *mainflux.Message) (err error) { defer func(begin time.Time) { message := fmt.Sprintf("Method save_states took %s to complete", time.Since(begin)) if err != nil { diff --git a/twins/api/metrics.go b/twins/api/metrics.go index a70e9c92f7..c88a7be8a6 100644 --- a/twins/api/metrics.go +++ b/twins/api/metrics.go @@ -10,7 +10,7 @@ import ( "time" "github.com/go-kit/kit/metrics" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/twins" ) @@ -68,7 +68,7 @@ func (ms *metricsMiddleware) ListTwins(ctx context.Context, token string, offset return ms.svc.ListTwins(ctx, token, offset, limit, name, metadata) } -func (ms *metricsMiddleware) SaveStates(msg *broker.Message) error { +func (ms *metricsMiddleware) SaveStates(msg *mainflux.Message) error { defer func(begin time.Time) { ms.counter.With("method", "save_states").Add(1) ms.latency.With("method", "save_states").Observe(time.Since(begin).Seconds()) diff --git a/twins/mocks/messages.go b/twins/mocks/messages.go index d9c2c9c5bc..bda1cd13c9 100644 --- a/twins/mocks/messages.go +++ b/twins/mocks/messages.go @@ -4,44 +4,26 @@ package mocks import ( - "context" - - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" - "github.com/nats-io/nats.go" ) -var _ broker.Nats = (*mockBroker)(nil) +var _ mainflux.Publisher = (*mockBroker)(nil) type mockBroker struct { subscriptions map[string]string } // New returns mock message publisher. -func New(sub map[string]string) broker.Nats { +func New(sub map[string]string) mainflux.Publisher { return &mockBroker{ subscriptions: sub, } } -func (mb mockBroker) Publish(_ context.Context, _ string, msg broker.Message) error { +func (mb mockBroker) Publish(topic string, msg mainflux.Message) error { if len(msg.Payload) == 0 { return errors.New("failed to publish") } return nil } - -func (mb mockBroker) Subscribe(subject string, f func(*nats.Msg)) (*nats.Subscription, error) { - if _, ok := mb.subscriptions[subject]; !ok { - return nil, errors.New("failed to subscribe") - } - - return nil, nil -} - -func (mb mockBroker) QueueSubscribe(chanID, subtopic string, f func(*nats.Msg)) (*nats.Subscription, error) { - return nil, nil -} - -func (mb mockBroker) Close() { -} diff --git a/twins/service.go b/twins/service.go index 66848f74d3..3fb30163c0 100644 --- a/twins/service.go +++ b/twins/service.go @@ -10,7 +10,6 @@ import ( "math" "time" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" @@ -70,7 +69,7 @@ type Service interface { ListStates(ctx context.Context, token string, offset uint64, limit uint64, id string) (StatesPage, error) // SaveStates persists states into database - SaveStates(msg *broker.Message) error + SaveStates(msg *mainflux.Message) error } const ( @@ -95,7 +94,7 @@ var crudOp = map[string]string{ } type twinsService struct { - broker broker.Nats + publisher mainflux.Publisher auth mainflux.AuthNServiceClient twins TwinRepository states StateRepository @@ -107,9 +106,9 @@ type twinsService struct { var _ Service = (*twinsService)(nil) // New instantiates the twins service implementation. -func New(broker broker.Nats, auth mainflux.AuthNServiceClient, twins TwinRepository, sr StateRepository, idp IdentityProvider, chann string, logger logger.Logger) Service { +func New(publisher mainflux.Publisher, auth mainflux.AuthNServiceClient, twins TwinRepository, sr StateRepository, idp IdentityProvider, chann string, logger logger.Logger) Service { return &twinsService{ - broker: broker, + publisher: publisher, auth: auth, twins: twins, states: sr, @@ -279,7 +278,7 @@ func (ts *twinsService) ListStates(ctx context.Context, token string, offset uin return ts.states.RetrieveAll(ctx, offset, limit, id) } -func (ts *twinsService) SaveStates(msg *broker.Message) error { +func (ts *twinsService) SaveStates(msg *mainflux.Message) error { ids, err := ts.twins.RetrieveByAttribute(context.TODO(), msg.Channel, msg.Subtopic) if err != nil { return err @@ -294,7 +293,7 @@ func (ts *twinsService) SaveStates(msg *broker.Message) error { return nil } -func (ts *twinsService) saveState(msg *broker.Message, id string) error { +func (ts *twinsService) saveState(msg *mainflux.Message, id string) error { var b []byte var err error defer ts.publish(&id, &err, crudOp["stateSucc"], crudOp["stateFail"], &b) @@ -336,7 +335,7 @@ func (ts *twinsService) saveState(msg *broker.Message, id string) error { return nil } -func prepareState(st *State, tw *Twin, rec senml.Record, msg *broker.Message) int { +func prepareState(st *State, tw *Twin, rec senml.Record, msg *mainflux.Message) int { def := tw.Definitions[len(tw.Definitions)-1] st.TwinID = tw.ID st.Definition = def.ID @@ -427,14 +426,15 @@ func (ts *twinsService) publish(twinID *string, err *error, succOp, failOp strin pl = []byte(fmt.Sprintf("{\"deleted\":\"%s\"}", *twinID)) } - mc := broker.Message{ + msg := mainflux.Message{ Channel: ts.channelID, Subtopic: op, Payload: pl, Publisher: publisher, + Occured: time.Now().UnixNano(), } - if err := ts.broker.Publish(context.TODO(), "", mc); err != nil { + if err := ts.publisher.Publish(msg.Channel, msg); err != nil { ts.logger.Warn(fmt.Sprintf("Failed to publish notification on NATS: %s", err)) } } From b599cc020b96fc6e4905e0cf8849dca5fcb3ee1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:12:00 +0200 Subject: [PATCH 08/28] Update LoRa adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/lora/main.go | 13 ++++++++----- lora/adapter.go | 20 +++++++------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/lora/main.go b/cmd/lora/main.go index bf941582b4..89346a9a75 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -15,11 +15,12 @@ import ( mqttPaho "github.com/eclipse/paho.mqtt.golang" r "github.com/go-redis/redis" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/lora" "github.com/mainflux/mainflux/lora/api" "github.com/mainflux/mainflux/lora/mqtt" + "github.com/mainflux/mainflux/nats" + broker "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux/lora/redis" @@ -85,19 +86,21 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() + + n := nats.New(nc, "", logger) thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) chanRM := newRouteMapRepositoy(rmConn, channelsRMPrefix, logger) mqttConn := connectToMQTTBroker(cfg.loraMsgURL, logger) - svc := lora.New(b, thingRM, chanRM) + svc := lora.New(n, thingRM, chanRM) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( svc, diff --git a/lora/adapter.go b/lora/adapter.go index ed6bf9fe50..5900f051e4 100644 --- a/lora/adapter.go +++ b/lora/adapter.go @@ -7,8 +7,7 @@ import ( "errors" "time" - "github.com/golang/protobuf/ptypes" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" ) const ( @@ -56,15 +55,15 @@ type Service interface { var _ Service = (*adapterService)(nil) type adapterService struct { - broker broker.Nats + publisher mainflux.Publisher thingsRM RouteMapRepository channelsRM RouteMapRepository } // New instantiates the LoRa adapter implementation. -func New(broker broker.Nats, thingsRM, channelsRM RouteMapRepository) Service { +func New(publisher mainflux.Publisher, thingsRM, channelsRM RouteMapRepository) Service { return &adapterService{ - broker: broker, + publisher: publisher, thingsRM: thingsRM, channelsRM: channelsRM, } @@ -101,21 +100,16 @@ func (as *adapterService) Publish(ctx context.Context, token string, m Message) payload = []byte(jo) } - created, err := ptypes.TimestampProto(time.Now()) - if err != nil { - return err - } - // Publish on Mainflux NATS broker - msg := broker.Message{ + msg := mainflux.Message{ Publisher: thing, Protocol: protocol, Channel: channel, Payload: payload, - Created: created, + Occured: time.Now().UnixNano(), } - return as.broker.Publish(ctx, token, msg) + return as.publisher.Publish(msg.Channel, msg) } func (as *adapterService) CreateThing(thingID string, devEUI string) error { From e4cd0c05e15cc562d3a9d758628c6cbc76993127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:14:01 +0200 Subject: [PATCH 09/28] Update OPC UA adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/opcua/main.go | 13 ++++++++----- http/adapter.go | 4 ++-- opcua/gopcua/subscribe.go | 20 +++++++------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index 4ab99429d5..92ae2a53ce 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -15,13 +15,14 @@ import ( r "github.com/go-redis/redis" "github.com/mainflux/mainflux" - "github.com/mainflux/mainflux/broker" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/opcua" "github.com/mainflux/mainflux/opcua/api" "github.com/mainflux/mainflux/opcua/db" "github.com/mainflux/mainflux/opcua/gopcua" "github.com/mainflux/mainflux/opcua/redis" + broker "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" @@ -97,15 +98,17 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - b, err := broker.New(cfg.natsURL) + nc, err := broker.Connect(cfg.natsURL) if err != nil { - logger.Error(err.Error()) + logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer b.Close() + defer nc.Close() + + n := nats.New(nc, "", logger) ctx := context.Background() - sub := gopcua.NewSubscriber(ctx, b, thingRM, chanRM, connRM, logger) + sub := gopcua.NewSubscriber(ctx, n, thingRM, chanRM, connRM, logger) browser := gopcua.NewBrowser(ctx, logger) svc := opcua.New(sub, browser, thingRM, chanRM, connRM, cfg.opcuaConfig, logger) diff --git a/http/adapter.go b/http/adapter.go index 874683fbcd..a1d7b4f4ba 100644 --- a/http/adapter.go +++ b/http/adapter.go @@ -35,7 +35,7 @@ func New(publisher mainflux.Publisher, things mainflux.ThingsServiceClient) Serv func (as *adapterService) Publish(ctx context.Context, token string, msg mainflux.Message) error { ar := &mainflux.AccessByKeyReq{ Token: token, - ChanID: msg.GetChannel(), + ChanID: msg.Channel, } thid, err := as.things.CanAccessByKey(ctx, ar) if err != nil { @@ -43,5 +43,5 @@ func (as *adapterService) Publish(ctx context.Context, token string, msg mainflu } msg.Publisher = thid.GetValue() - return as.publisher.Publish(msg) + return as.publisher.Publish(msg.Channel, msg) } diff --git a/opcua/gopcua/subscribe.go b/opcua/gopcua/subscribe.go index 50f2f6916c..5ad777d073 100644 --- a/opcua/gopcua/subscribe.go +++ b/opcua/gopcua/subscribe.go @@ -9,10 +9,9 @@ import ( "strconv" "time" - "github.com/golang/protobuf/ptypes" opcuaGopcua "github.com/gopcua/opcua" uaGopcua "github.com/gopcua/opcua/ua" - "github.com/mainflux/mainflux/broker" + "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/opcua" @@ -41,7 +40,7 @@ var _ opcua.Subscriber = (*client)(nil) type client struct { ctx context.Context - broker broker.Nats + publisher mainflux.Publisher thingsRM opcua.RouteMapRepository channelsRM opcua.RouteMapRepository connectRM opcua.RouteMapRepository @@ -58,10 +57,10 @@ type message struct { } // NewSubscriber returns new OPC-UA client instance. -func NewSubscriber(ctx context.Context, broker broker.Nats, thingsRM, channelsRM, connectRM opcua.RouteMapRepository, log logger.Logger) opcua.Subscriber { +func NewSubscriber(ctx context.Context, publisher mainflux.Publisher, thingsRM, channelsRM, connectRM opcua.RouteMapRepository, log logger.Logger) opcua.Subscriber { return client{ ctx: ctx, - broker: broker, + publisher: publisher, thingsRM: thingsRM, channelsRM: channelsRM, connectRM: connectRM, @@ -225,24 +224,19 @@ func (c client) publish(token string, m message) error { return fmt.Errorf("%s between channel %s and thing %s", errNotFoundConn, chanID, thingID) } - created, err := ptypes.TimestampProto(time.Now()) - if err != nil { - return err - } - // Publish on Mainflux NATS broker SenML := fmt.Sprintf(`[{"n":"%s", "t": %d, "%s":%v}]`, m.Type, m.Time, m.DataKey, m.Data) payload := []byte(SenML) - msg := broker.Message{ + msg := mainflux.Message{ Publisher: thingID, Protocol: protocol, Channel: chanID, Payload: payload, Subtopic: m.NodeID, - Created: created, + Occured: time.Now().UnixNano(), } - if err := c.broker.Publish(c.ctx, token, msg); err != nil { + if err := c.publisher.Publish(msg.Channel, msg); err != nil { return err } From f469a83c3fdad0deea9ae94da83ab7de480e3ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:17:50 +0200 Subject: [PATCH 10/28] Remove broker package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Package `broker` is conceptually renamed to package `nats`. Signed-off-by: Dušan Borovčanin --- broker/message.pb.go | 124 ------------------------------------------- broker/message.proto | 16 ------ broker/nats.go | 100 ---------------------------------- 3 files changed, 240 deletions(-) delete mode 100644 broker/message.pb.go delete mode 100644 broker/message.proto delete mode 100644 broker/nats.go diff --git a/broker/message.pb.go b/broker/message.pb.go deleted file mode 100644 index 46cf0ba410..0000000000 --- a/broker/message.pb.go +++ /dev/null @@ -1,124 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: broker/message.proto - -package broker - -import ( - fmt "fmt" - proto "github.com/gogo/protobuf/proto" - timestamp "github.com/golang/protobuf/ptypes/timestamp" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -// Message represents a message emitted by the Mainflux adapters layer. -type Message struct { - Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` - Subtopic string `protobuf:"bytes,2,opt,name=subtopic,proto3" json:"subtopic,omitempty"` - Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` - Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` - Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` - Created *timestamp.Timestamp `protobuf:"bytes,6,opt,name=created,proto3" json:"created,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Message) Reset() { *m = Message{} } -func (m *Message) String() string { return proto.CompactTextString(m) } -func (*Message) ProtoMessage() {} -func (*Message) Descriptor() ([]byte, []int) { - return fileDescriptor_6357da820a7eacc2, []int{0} -} -func (m *Message) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Message.Unmarshal(m, b) -} -func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Message.Marshal(b, m, deterministic) -} -func (m *Message) XXX_Merge(src proto.Message) { - xxx_messageInfo_Message.Merge(m, src) -} -func (m *Message) XXX_Size() int { - return xxx_messageInfo_Message.Size(m) -} -func (m *Message) XXX_DiscardUnknown() { - xxx_messageInfo_Message.DiscardUnknown(m) -} - -var xxx_messageInfo_Message proto.InternalMessageInfo - -func (m *Message) GetChannel() string { - if m != nil { - return m.Channel - } - return "" -} - -func (m *Message) GetSubtopic() string { - if m != nil { - return m.Subtopic - } - return "" -} - -func (m *Message) GetPublisher() string { - if m != nil { - return m.Publisher - } - return "" -} - -func (m *Message) GetProtocol() string { - if m != nil { - return m.Protocol - } - return "" -} - -func (m *Message) GetPayload() []byte { - if m != nil { - return m.Payload - } - return nil -} - -func (m *Message) GetCreated() *timestamp.Timestamp { - if m != nil { - return m.Created - } - return nil -} - -func init() { - proto.RegisterType((*Message)(nil), "broker.Message") -} - -func init() { proto.RegisterFile("broker/message.proto", fileDescriptor_6357da820a7eacc2) } - -var fileDescriptor_6357da820a7eacc2 = []byte{ - // 202 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x8f, 0xcb, 0x6e, 0x84, 0x20, - 0x18, 0x85, 0x43, 0x2f, 0x5a, 0x69, 0x57, 0xa4, 0x0b, 0x62, 0x9a, 0xd4, 0x74, 0xe5, 0x0a, 0x93, - 0xb6, 0xaf, 0x31, 0x1b, 0x33, 0x2f, 0x00, 0xf8, 0x8f, 0x9a, 0x41, 0x7f, 0x02, 0xb8, 0x98, 0x47, - 0x9c, 0xb7, 0x9a, 0x08, 0x61, 0x66, 0xf9, 0x9d, 0x4b, 0x4e, 0x0e, 0xfd, 0x54, 0x0e, 0xcf, 0xe0, - 0xba, 0x05, 0xbc, 0x97, 0x23, 0x08, 0xeb, 0x30, 0x20, 0x2b, 0x92, 0x5a, 0x7f, 0x8f, 0x88, 0xa3, - 0x81, 0x2e, 0xaa, 0x6a, 0x3b, 0x75, 0x61, 0x5e, 0xc0, 0x07, 0xb9, 0xd8, 0x14, 0xfc, 0xb9, 0x12, - 0x5a, 0x1e, 0x52, 0x95, 0x71, 0x5a, 0xea, 0x49, 0xae, 0x2b, 0x18, 0x4e, 0x1a, 0xd2, 0x56, 0x7d, - 0x46, 0x56, 0xd3, 0x37, 0xbf, 0xa9, 0x80, 0x76, 0xd6, 0xfc, 0x29, 0x5a, 0x77, 0x66, 0x5f, 0xb4, - 0xb2, 0x9b, 0x32, 0xb3, 0x9f, 0xc0, 0xf1, 0xe7, 0x68, 0x3e, 0x84, 0xbd, 0x19, 0x87, 0x34, 0x1a, - 0xfe, 0x92, 0x9a, 0x99, 0xf7, 0x3d, 0x2b, 0x2f, 0x06, 0xe5, 0xc0, 0x5f, 0x1b, 0xd2, 0x7e, 0xf4, - 0x19, 0xd9, 0x3f, 0x2d, 0xb5, 0x03, 0x19, 0x60, 0xe0, 0x45, 0x43, 0xda, 0xf7, 0xdf, 0x5a, 0xa4, - 0x23, 0x22, 0x1f, 0x11, 0xc7, 0x7c, 0xa4, 0xcf, 0x51, 0x55, 0x44, 0xf3, 0xef, 0x16, 0x00, 0x00, - 0xff, 0xff, 0x0b, 0xd8, 0x04, 0x71, 0x13, 0x01, 0x00, 0x00, -} diff --git a/broker/message.proto b/broker/message.proto deleted file mode 100644 index b276e9b6f0..0000000000 --- a/broker/message.proto +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Mainflux -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; -package broker; -import "google/protobuf/timestamp.proto"; - -// Message represents a message emitted by the Mainflux adapters layer. -message Message { - string channel = 1; - string subtopic = 2; - string publisher = 3; - string protocol = 4; - bytes payload = 5; - google.protobuf.Timestamp created = 6; -} diff --git a/broker/nats.go b/broker/nats.go deleted file mode 100644 index fdc2689b86..0000000000 --- a/broker/nats.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) Mainflux -// SPDX-License-Identifier: Apache-2.0 - -package broker - -import ( - "context" - "fmt" - - "github.com/gogo/protobuf/proto" - "github.com/mainflux/mainflux/errors" - "github.com/nats-io/nats.go" -) - -// Nats specifies a NATS message API. -type Nats interface { - // Publish publishes message to the msessage broker. - Publish(ctx context.Context, token string, msg Message) error - - // Subscribe subscribes to a message broker subject. - Subscribe(subject string, consumer func(msg *nats.Msg)) (*nats.Subscription, error) - - // Subscribe subscribes to the message broker for a given channel ID and subtopic. - QueueSubscribe(subject, queue string, f func(msg *nats.Msg)) (*nats.Subscription, error) - - // Close closes NATS connection. - Close() -} - -const ( - chansPrefix = "channels" - - // SubjectAllChannels define the subject to subscribe to all channels messages - SubjectAllChannels = "channels.>" -) - -var ( - errNatsConn = errors.New("Failed to connect to NATS") - errNatsPub = errors.New("Failed to publish to NATS") - errNatsSub = errors.New("Failed to subscribe to NATS") - errNatsQueueSub = errors.New("Failed to queue subscribe to NATS") -) - -var _ Nats = (*broker)(nil) - -type broker struct { - conn *nats.Conn -} - -// New returns NATS message broker. -func New(url string) (Nats, error) { - nc, err := nats.Connect(url) - if err != nil { - return nil, errors.Wrap(errNatsConn, err) - } - - return &broker{ - conn: nc, - }, nil -} - -func (b broker) Publish(_ context.Context, _ string, msg Message) error { - data, err := proto.Marshal(&msg) - if err != nil { - return err - } - - subject := fmt.Sprintf("%s.%s", chansPrefix, msg.Channel) - if msg.Subtopic != "" { - subject = fmt.Sprintf("%s.%s", subject, msg.Subtopic) - } - if err := b.conn.Publish(subject, data); err != nil { - return errors.Wrap(errNatsPub, err) - } - - return nil -} - -func (b broker) Subscribe(subject string, f func(msg *nats.Msg)) (*nats.Subscription, error) { - ps := fmt.Sprintf("%s.%s", chansPrefix, subject) - sub, err := b.conn.Subscribe(ps, f) - if err != nil { - return nil, errors.Wrap(errNatsSub, err) - } - - return sub, nil -} - -func (b broker) QueueSubscribe(subject, queue string, f func(msg *nats.Msg)) (*nats.Subscription, error) { - sub, err := b.conn.QueueSubscribe(subject, queue, f) - if err != nil { - return nil, errors.Wrap(errNatsQueueSub, err) - } - - return sub, nil -} - -func (b broker) Close() { - b.conn.Close() -} From bd5bec34b0c9d0110f14b51418501a40b3d827a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:28:13 +0200 Subject: [PATCH 11/28] Update makefile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index c3c890512a..6af40c34f0 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,6 @@ test: proto: protoc --gofast_out=plugins=grpc:. *.proto - protoc --gogo_out=plugins=grpc:. broker/*.proto $(SERVICES): $(call compile_service,$(@)) From 244a27bd429eb08106cfc870e83e20c79531b771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:38:03 +0200 Subject: [PATCH 12/28] Add comment explanation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- nats/pubsub.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nats/pubsub.go b/nats/pubsub.go index 15e1cfe87c..9115c96423 100644 --- a/nats/pubsub.go +++ b/nats/pubsub.go @@ -36,6 +36,10 @@ type nats struct { } // New returns NATS message broker. +// Paramter queue specifies the queue for the Subscribe method. If queue is specified (is not an empty string), +// Subscribe method will execute NATS QueueSubscibe which is conceptually different from ordinary subscribe. +// For more information, please take a look here: https://docs.nats.io/developing-with-nats/receiving/queues. +// If the queue is empty, Subscribe will be used. func New(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { return &nats{ conn: conn, From a55fff566624086400ebdec43799c13c29035396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 00:39:18 +0200 Subject: [PATCH 13/28] Fix MQTT adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/mqtt/main.go | 4 ++-- mqtt/adapter.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index 0a5f5709e5..1520dd2c40 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -137,12 +137,12 @@ func main() { } defer nc.Close() - b := nats.New(nc, "", "", logger) + n := nats.New(nc, "", logger) es := mr.NewEventStore(rc, cfg.instance) // Event handler for MQTT hooks - evt := mqtt.New([]mainflux.Publisher{b}, cc, es, logger, tracer) + evt := mqtt.New([]mainflux.Publisher{n}, cc, es, logger, tracer) errs := make(chan error, 2) diff --git a/mqtt/adapter.go b/mqtt/adapter.go index 6f7e3a2751..6cfd19b9df 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -47,11 +47,11 @@ type handler struct { func New(publishers []mainflux.Publisher, tc mainflux.ThingsServiceClient, es redis.EventStore, logger logger.Logger, tracer opentracing.Tracer) session.Handler { return &handler{ - publishers: publishers, tc: tc, es: es, tracer: tracer, logger: logger, + publishers: publishers, } } @@ -159,7 +159,7 @@ func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { } for _, pub := range h.publishers { - if err := pub.Publish(msg); err != nil { + if err := pub.Publish(msg.Channel, msg); err != nil { h.logger.Info("Error publishing to Mainflux " + err.Error()) } } From ccd15f3edf2898c0bf9b73e74022540e069f0a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 11:44:35 +0200 Subject: [PATCH 14/28] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- coap/api/transport.go | 2 +- http/api/transport.go | 2 +- lora/adapter.go | 2 +- message.pb.go | 44 ++++++++++++++++++++------------------- message.proto | 4 +++- mqtt/adapter.go | 2 +- opcua/gopcua/subscribe.go | 2 +- twins/service.go | 2 +- 8 files changed, 32 insertions(+), 28 deletions(-) diff --git a/coap/api/transport.go b/coap/api/transport.go index ff242af7a8..103374b5d6 100644 --- a/coap/api/transport.go +++ b/coap/api/transport.go @@ -229,7 +229,7 @@ func receive(svc coap.Service, msg *gocoap.Message) *gocoap.Message { Publisher: publisher, Protocol: protocol, Payload: msg.Payload, - Occured: time.Now().UnixNano(), + Occurred: time.Now().UnixNano(), } if err := svc.Publish(m); err != nil { diff --git a/http/api/transport.go b/http/api/transport.go index 53b2dc2512..72d72e22a1 100644 --- a/http/api/transport.go +++ b/http/api/transport.go @@ -113,7 +113,7 @@ func decodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { Channel: chanID, Subtopic: subtopic, Payload: payload, - Occured: time.Now().UnixNano(), + Occurred: time.Now().UnixNano(), } req := publishReq{ diff --git a/lora/adapter.go b/lora/adapter.go index 5900f051e4..0fad3417d7 100644 --- a/lora/adapter.go +++ b/lora/adapter.go @@ -106,7 +106,7 @@ func (as *adapterService) Publish(ctx context.Context, token string, m Message) Protocol: protocol, Channel: channel, Payload: payload, - Occured: time.Now().UnixNano(), + Occurred: time.Now().UnixNano(), } return as.publisher.Publish(msg.Channel, msg) diff --git a/message.pb.go b/message.pb.go index 4a67eb2c11..1e3194fdd6 100644 --- a/message.pb.go +++ b/message.pb.go @@ -24,12 +24,14 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package // Message represents a message emitted by the Mainflux adapters layer. type Message struct { - Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` - Subtopic string `protobuf:"bytes,2,opt,name=subtopic,proto3" json:"subtopic,omitempty"` - Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` - Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` - Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` - Occured int64 `protobuf:"varint,6,opt,name=occured,proto3" json:"occured,omitempty"` + Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` + Subtopic string `protobuf:"bytes,2,opt,name=subtopic,proto3" json:"subtopic,omitempty"` + Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` + Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` + Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` + // Timestamp the Message occured in the system. + // By the default, occured represents Unix nanoseconds timestamp. + Occurred int64 `protobuf:"varint,6,opt,name=occurred,proto3" json:"occurred,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -103,9 +105,9 @@ func (m *Message) GetPayload() []byte { return nil } -func (m *Message) GetOccured() int64 { +func (m *Message) GetOccurred() int64 { if m != nil { - return m.Occured + return m.Occurred } return 0 } @@ -117,19 +119,19 @@ func init() { func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 186 bytes of a gzipped FileDescriptorProto + // 187 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xc8, 0x4d, 0xcc, 0xcc, 0x4b, - 0xcb, 0x29, 0xad, 0x50, 0x5a, 0xcb, 0xc8, 0xc5, 0xee, 0x0b, 0x91, 0x13, 0x92, 0xe0, 0x62, 0x4f, + 0xcb, 0x29, 0xad, 0x50, 0x5a, 0xcf, 0xc8, 0xc5, 0xee, 0x0b, 0x91, 0x13, 0x92, 0xe0, 0x62, 0x4f, 0xce, 0x48, 0xcc, 0xcb, 0x4b, 0xcd, 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0x71, 0x85, 0xa4, 0xb8, 0x38, 0x8a, 0x4b, 0x93, 0x4a, 0xf2, 0x0b, 0x32, 0x93, 0x25, 0x98, 0xc0, 0x52, 0x70, 0xbe, 0x90, 0x0c, 0x17, 0x67, 0x41, 0x69, 0x52, 0x4e, 0x66, 0x71, 0x46, 0x6a, 0x91, 0x04, 0x33, 0x58, 0x12, 0x21, 0x00, 0xd2, 0x09, 0xb6, 0x32, 0x39, 0x3f, 0x47, 0x82, 0x05, 0xa2, 0x13, 0xc6, 0x07, 0xd9, 0x57, 0x90, 0x58, 0x99, 0x93, 0x9f, 0x98, 0x22, 0xc1, 0xaa, 0xc0, 0xa8, 0xc1, 0x13, - 0x04, 0xe3, 0x82, 0x64, 0xf2, 0x93, 0x93, 0x4b, 0x8b, 0x52, 0x53, 0x24, 0xd8, 0x14, 0x18, 0x35, - 0x98, 0x83, 0x60, 0x5c, 0x27, 0x81, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, - 0x48, 0x8e, 0x71, 0xc6, 0x63, 0x39, 0x86, 0x24, 0x36, 0xb0, 0x79, 0xc6, 0x80, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xe5, 0x53, 0xe6, 0x15, 0xe3, 0x00, 0x00, 0x00, + 0x04, 0xe3, 0x82, 0x74, 0xe5, 0x27, 0x27, 0x97, 0x16, 0x15, 0xa5, 0xa6, 0x48, 0xb0, 0x29, 0x30, + 0x6a, 0x30, 0x07, 0xc1, 0xf9, 0x4e, 0x02, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, + 0xe0, 0x91, 0x1c, 0xe3, 0x8c, 0xc7, 0x72, 0x0c, 0x49, 0x6c, 0x60, 0x13, 0x8d, 0x01, 0x01, 0x00, + 0x00, 0xff, 0xff, 0x81, 0x36, 0x5f, 0x25, 0xe5, 0x00, 0x00, 0x00, } func (m *Message) Marshal() (dAtA []byte, err error) { @@ -156,8 +158,8 @@ func (m *Message) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.Occured != 0 { - i = encodeVarintMessage(dAtA, i, uint64(m.Occured)) + if m.Occurred != 0 { + i = encodeVarintMessage(dAtA, i, uint64(m.Occurred)) i-- dAtA[i] = 0x30 } @@ -236,8 +238,8 @@ func (m *Message) Size() (n int) { if l > 0 { n += 1 + l + sovMessage(uint64(l)) } - if m.Occured != 0 { - n += 1 + sovMessage(uint64(m.Occured)) + if m.Occurred != 0 { + n += 1 + sovMessage(uint64(m.Occurred)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -444,9 +446,9 @@ func (m *Message) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 6: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Occured", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Occurred", wireType) } - m.Occured = 0 + m.Occurred = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMessage @@ -456,7 +458,7 @@ func (m *Message) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Occured |= int64(b&0x7F) << shift + m.Occurred |= int64(b&0x7F) << shift if b < 0x80 { break } diff --git a/message.proto b/message.proto index 07ec5a38e1..ce048434af 100644 --- a/message.proto +++ b/message.proto @@ -11,5 +11,7 @@ message Message { string publisher = 3; string protocol = 4; bytes payload = 5; - int64 occured = 6; + // Timestamp the Message occured in the system. + // By the default, occured represents Unix nanoseconds timestamp. + int64 occurred = 6; } diff --git a/mqtt/adapter.go b/mqtt/adapter.go index 6cfd19b9df..4c8c31dbea 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -155,7 +155,7 @@ func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { Subtopic: subtopic, Publisher: c.Username, Payload: *payload, - Occured: time.Now().UnixNano(), + Occurred: time.Now().UnixNano(), } for _, pub := range h.publishers { diff --git a/opcua/gopcua/subscribe.go b/opcua/gopcua/subscribe.go index 5ad777d073..1d7adbb689 100644 --- a/opcua/gopcua/subscribe.go +++ b/opcua/gopcua/subscribe.go @@ -233,7 +233,7 @@ func (c client) publish(token string, m message) error { Channel: chanID, Payload: payload, Subtopic: m.NodeID, - Occured: time.Now().UnixNano(), + Occurred: time.Now().UnixNano(), } if err := c.publisher.Publish(msg.Channel, msg); err != nil { diff --git a/twins/service.go b/twins/service.go index 3fb30163c0..cbd858772b 100644 --- a/twins/service.go +++ b/twins/service.go @@ -431,7 +431,7 @@ func (ts *twinsService) publish(twinID *string, err *error, succOp, failOp strin Subtopic: op, Payload: pl, Publisher: publisher, - Occured: time.Now().UnixNano(), + Occurred: time.Now().UnixNano(), } if err := ts.publisher.Publish(msg.Channel, msg); err != nil { From a88c9cb1986a297dc54d5f8cc80f6ea56e76bdc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 16:48:06 +0200 Subject: [PATCH 15/28] Move NATS pub/sub implementation to pubsub pkg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 10 ++++---- cmd/coap/main.go | 6 ++--- cmd/http/main.go | 10 ++++---- cmd/influxdb-writer/main.go | 10 ++++---- cmd/lora/main.go | 10 ++++---- cmd/mongodb-writer/main.go | 10 ++++---- cmd/mqtt/main.go | 10 ++++---- cmd/opcua/main.go | 10 ++++---- cmd/postgres-writer/main.go | 10 ++++---- cmd/twins/main.go | 10 ++++---- pubsub/nats/doc.go | 11 +++++++++ pubsub/nats/pubsliher.go | 42 +++++++++++++++++++++++++++++++++ {nats => pubsub/nats}/pubsub.go | 26 ++++++++++---------- writers/writer.go | 6 ++--- 14 files changed, 118 insertions(+), 63 deletions(-) create mode 100644 pubsub/nats/doc.go create mode 100644 pubsub/nats/pubsliher.go rename {nats => pubsub/nats}/pubsub.go (74%) diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 386e9ac33e..358599e136 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -17,12 +17,12 @@ import ( "github.com/gocql/gocql" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/cassandra" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -70,21 +70,21 @@ func main() { log.Fatalf(err.Error()) } - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + ps := pubsub.NewPubSub(nc, "", logger) session := connectToCassandra(cfg.dbCfg, logger) defer session.Close() repo := newService(session, logger) st := senml.New(cfg.contentType) - if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Cassandra writer: %s", err)) } diff --git a/cmd/coap/main.go b/cmd/coap/main.go index 94b9ca731f..b0688c3be9 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -21,7 +21,7 @@ import ( "github.com/mainflux/mainflux/coap" "github.com/mainflux/mainflux/coap/api" logger "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" broker "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" @@ -89,9 +89,9 @@ func main() { } defer nc.Close() - n := nats.New(nc, "", logger) + ps := pubsub.NewPubSub(nc, "", logger) - svc := coap.New(n, logger, cc, respChan) + svc := coap.New(ps, logger, cc, respChan) svc = api.LoggingMiddleware(svc, logger) diff --git a/cmd/http/main.go b/cmd/http/main.go index 7eab69db97..9b7220ec0d 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -22,9 +22,9 @@ import ( adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/http/api" "github.com/mainflux/mainflux/logger" - nats "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -79,17 +79,17 @@ func main() { thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) defer thingsCloser.Close() - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + pub := pubsub.NewPublisher(nc) tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - svc := adapter.New(n, tc) + svc := adapter.New(pub, tc) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index 02a74afa1d..c0739afa4a 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -15,12 +15,12 @@ import ( influxdata "github.com/influxdata/influxdb/client/v2" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/influxdb" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -71,14 +71,14 @@ func main() { log.Fatalf(err.Error()) } - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + ps := pubsub.NewPubSub(nc, "", logger) client, err := influxdata.NewHTTPClient(clientCfg) if err != nil { @@ -94,7 +94,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start InfluxDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/lora/main.go b/cmd/lora/main.go index 89346a9a75..1887a476df 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -19,8 +19,8 @@ import ( "github.com/mainflux/mainflux/lora" "github.com/mainflux/mainflux/lora/api" "github.com/mainflux/mainflux/lora/mqtt" - "github.com/mainflux/mainflux/nats" - broker "github.com/nats-io/nats.go" + pubsub "github.com/mainflux/mainflux/pubsub/nats" + "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux/lora/redis" @@ -86,21 +86,21 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + pub := pubsub.NewPublisher(nc) thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) chanRM := newRouteMapRepositoy(rmConn, channelsRMPrefix, logger) mqttConn := connectToMQTTBroker(cfg.loraMsgURL, logger) - svc := lora.New(n, thingRM, chanRM) + svc := lora.New(pub, thingRM, chanRM) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( svc, diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 9750f71dc2..53e56a92c0 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -15,12 +15,12 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/mongodb" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -67,14 +67,14 @@ func main() { log.Fatal(err) } - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + ps := pubsub.NewPubSub(nc, "", logger) addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) @@ -91,7 +91,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start MongoDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index 1520dd2c40..bb437b71ab 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -17,12 +17,12 @@ import ( "github.com/mainflux/mainflux/logger" mqtt "github.com/mainflux/mainflux/mqtt" mr "github.com/mainflux/mainflux/mqtt/redis" - nats "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" mp "github.com/mainflux/mproxy/pkg/mqtt" "github.com/mainflux/mproxy/pkg/session" ws "github.com/mainflux/mproxy/pkg/websocket" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" jconfig "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" @@ -130,19 +130,19 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + pub := pubsub.NewPublisher(nc) es := mr.NewEventStore(rc, cfg.instance) // Event handler for MQTT hooks - evt := mqtt.New([]mainflux.Publisher{n}, cc, es, logger, tracer) + evt := mqtt.New([]mainflux.Publisher{pub}, cc, es, logger, tracer) errs := make(chan error, 2) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index 92ae2a53ce..126014f335 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -16,13 +16,13 @@ import ( r "github.com/go-redis/redis" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" "github.com/mainflux/mainflux/opcua" "github.com/mainflux/mainflux/opcua/api" "github.com/mainflux/mainflux/opcua/db" "github.com/mainflux/mainflux/opcua/gopcua" "github.com/mainflux/mainflux/opcua/redis" - broker "github.com/nats-io/nats.go" + pubsub "github.com/mainflux/mainflux/pubsub/nats" + "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" @@ -98,17 +98,17 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + ps := pubsub.NewPubSub(nc, "", logger) ctx := context.Background() - sub := gopcua.NewSubscriber(ctx, n, thingRM, chanRM, connRM, logger) + sub := gopcua.NewSubscriber(ctx, ps, thingRM, chanRM, connRM, logger) browser := gopcua.NewBrowser(ctx, logger) svc := opcua.New(sub, browser, thingRM, chanRM, connRM, cfg.opcuaConfig, logger) diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index f17a3e315a..42d36ef386 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -15,12 +15,12 @@ import ( "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/postgres" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -76,21 +76,21 @@ func main() { log.Fatalf(err.Error()) } - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, "", logger) + ps := pubsub.NewPubSub(nc, "", logger) db := connectToDB(cfg.dbConfig, logger) defer db.Close() repo := newService(db, logger) st := senml.New(cfg.contentType) - if err = writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err = writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) } diff --git a/cmd/twins/main.go b/cmd/twins/main.go index b608515aeb..e8d4f570db 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -19,14 +19,14 @@ import ( "github.com/mainflux/mainflux" authapi "github.com/mainflux/mainflux/authn/api/grpc" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" localusers "github.com/mainflux/mainflux/things/users" "github.com/mainflux/mainflux/twins" "github.com/mainflux/mainflux/twins/api" twapi "github.com/mainflux/mainflux/twins/api/http" twmongodb "github.com/mainflux/mainflux/twins/mongodb" "github.com/mainflux/mainflux/twins/uuid" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -114,14 +114,14 @@ func main() { dbTracer, dbCloser := initJaeger("twins_db", cfg.jaegerURL, logger) defer dbCloser.Close() - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } defer nc.Close() - n := nats.New(nc, queue, logger) + ps := pubsub.NewPubSub(nc, queue, logger) ncTracer, ncCloser := initJaeger("twins_nats", cfg.jaegerURL, logger) defer ncCloser.Close() @@ -129,7 +129,7 @@ func main() { tracer, closer := initJaeger("twins", cfg.jaegerURL, logger) defer closer.Close() - svc := newService(n, ncTracer, cfg.channelID, auth, dbTracer, db, logger) + svc := newService(ps, ncTracer, cfg.channelID, auth, dbTracer, db, logger) errs := make(chan error, 2) diff --git a/pubsub/nats/doc.go b/pubsub/nats/doc.go new file mode 100644 index 0000000000..ca3cf9d8c6 --- /dev/null +++ b/pubsub/nats/doc.go @@ -0,0 +1,11 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +// Package nats hold the implementation of the Publisher and PubSub +// interfaces for the NATS messaging system, the internal messaging +// broker of the Mainflux IoT platform. Due to the practical requirements +// implementation Publisher is created alongside PubSub. The reason for +// this is that Subscriber implementation of NATS brings the burden of +// additional struct fields which are not used by Publisher. Subscriber +// is not implemented separately because PubSub can be used where Subscriber is needed. +package nats diff --git a/pubsub/nats/pubsliher.go b/pubsub/nats/pubsliher.go new file mode 100644 index 0000000000..33c19f60e6 --- /dev/null +++ b/pubsub/nats/pubsliher.go @@ -0,0 +1,42 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +package nats + +import ( + "fmt" + + "github.com/gogo/protobuf/proto" + "github.com/mainflux/mainflux" + broker "github.com/nats-io/nats.go" +) + +var _ mainflux.PubSub = (*pubsub)(nil) + +type pubsliher struct { + conn *broker.Conn +} + +// NewPublisher returns NATS message Publisher. +func NewPublisher(conn *broker.Conn) mainflux.Publisher { + return &pubsub{ + conn: conn, + } +} + +func (pub *pubsliher) Publish(topic string, msg mainflux.Message) error { + data, err := proto.Marshal(&msg) + if err != nil { + return err + } + + subject := fmt.Sprintf("%s.%s", chansPrefix, topic) + if msg.Subtopic != "" { + subject = fmt.Sprintf("%s.%s", subject, msg.Subtopic) + } + if err := pub.conn.Publish(subject, data); err != nil { + return err + } + + return nil +} diff --git a/nats/pubsub.go b/pubsub/nats/pubsub.go similarity index 74% rename from nats/pubsub.go rename to pubsub/nats/pubsub.go index 9115c96423..b06718fd6c 100644 --- a/nats/pubsub.go +++ b/pubsub/nats/pubsub.go @@ -25,9 +25,9 @@ var ( errEmptyTopic = errors.New("empty topic") ) -var _ mainflux.PubSub = (*nats)(nil) +var _ mainflux.PubSub = (*pubsub)(nil) -type nats struct { +type pubsub struct { conn *broker.Conn logger log.Logger mu sync.Mutex @@ -35,13 +35,15 @@ type nats struct { subscriptions map[string]*broker.Subscription } -// New returns NATS message broker. -// Paramter queue specifies the queue for the Subscribe method. If queue is specified (is not an empty string), -// Subscribe method will execute NATS QueueSubscibe which is conceptually different from ordinary subscribe. -// For more information, please take a look here: https://docs.nats.io/developing-with-nats/receiving/queues. +// NewPubSub returns NATS message publisher/subscriber. +// Paramter queue specifies the queue for the Subscribe method. +// If queue is specified (is not an empty string), Subscribe method +// will execute NATS QueueSubscibe which is conceptually different +// from ordinary subscribe. For more information, please take a look +// here: https://docs.nats.io/developing-with-nats/receiving/queues. // If the queue is empty, Subscribe will be used. -func New(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { - return &nats{ +func NewPubSub(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { + return &pubsub{ conn: conn, queue: queue, logger: logger, @@ -49,7 +51,7 @@ func New(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { } } -func (n *nats) Publish(topic string, msg mainflux.Message) error { +func (n *pubsub) Publish(topic string, msg mainflux.Message) error { data, err := proto.Marshal(&msg) if err != nil { return err @@ -66,7 +68,7 @@ func (n *nats) Publish(topic string, msg mainflux.Message) error { return nil } -func (n *nats) Subscribe(topic string, handler mainflux.MessageHandler) error { +func (n *pubsub) Subscribe(topic string, handler mainflux.MessageHandler) error { if topic == "" { return errEmptyTopic } @@ -92,7 +94,7 @@ func (n *nats) Subscribe(topic string, handler mainflux.MessageHandler) error { return nil } -func (n *nats) Unsubscribe(topic string) error { +func (n *pubsub) Unsubscribe(topic string) error { if topic == "" { return errEmptyTopic } @@ -114,7 +116,7 @@ func (n *nats) Unsubscribe(topic string) error { return nil } -func (n *nats) natsHandler(h mainflux.MessageHandler) broker.MsgHandler { +func (n *pubsub) natsHandler(h mainflux.MessageHandler) broker.MsgHandler { return func(m *broker.Msg) { var msg mainflux.Message if err := proto.Unmarshal(m.Data, &msg); err != nil { diff --git a/writers/writer.go b/writers/writer.go index 2a50886e2a..69c0574d1e 100644 --- a/writers/writer.go +++ b/writers/writer.go @@ -11,7 +11,7 @@ import ( "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" - "github.com/mainflux/mainflux/nats" + pubsub "github.com/mainflux/mainflux/pubsub/nats" "github.com/mainflux/mainflux/transformers" "github.com/mainflux/mainflux/transformers/senml" ) @@ -75,12 +75,12 @@ type subjectsConfig struct { func loadSubjectsConfig(subjectsConfigPath string) ([]string, error) { data, err := ioutil.ReadFile(subjectsConfigPath) if err != nil { - return []string{nats.SubjectAllChannels}, errors.Wrap(errOpenConfFile, err) + return []string{pubsub.SubjectAllChannels}, errors.Wrap(errOpenConfFile, err) } var subjectsCfg subjectsConfig if err := toml.Unmarshal(data, &subjectsCfg); err != nil { - return []string{nats.SubjectAllChannels}, errors.Wrap(errParseConfFile, err) + return []string{pubsub.SubjectAllChannels}, errors.Wrap(errParseConfFile, err) } return subjectsCfg.Subjects.List, nil From 45b140751217b34523bc6e7fe63c2d3bca74f3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 16:51:14 +0200 Subject: [PATCH 16/28] Remove an empty line in main methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 1 - cmd/coap/main.go | 1 - cmd/http/main.go | 1 - cmd/influxdb-writer/main.go | 1 - cmd/lora/main.go | 1 - cmd/mongodb-writer/main.go | 1 - cmd/mqtt/main.go | 1 - cmd/opcua/main.go | 1 - cmd/postgres-writer/main.go | 1 - cmd/twins/main.go | 1 - 10 files changed, 10 deletions(-) diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 358599e136..8a6820bfbd 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -76,7 +76,6 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) session := connectToCassandra(cfg.dbCfg, logger) diff --git a/cmd/coap/main.go b/cmd/coap/main.go index b0688c3be9..67a2fd7731 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -90,7 +90,6 @@ func main() { defer nc.Close() ps := pubsub.NewPubSub(nc, "", logger) - svc := coap.New(ps, logger, cc, respChan) svc = api.LoggingMiddleware(svc, logger) diff --git a/cmd/http/main.go b/cmd/http/main.go index 9b7220ec0d..8fe057448b 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -85,7 +85,6 @@ func main() { os.Exit(1) } defer nc.Close() - pub := pubsub.NewPublisher(nc) tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index c0739afa4a..a8d42a7038 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -77,7 +77,6 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) client, err := influxdata.NewHTTPClient(clientCfg) diff --git a/cmd/lora/main.go b/cmd/lora/main.go index 1887a476df..ca7b3dea60 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -92,7 +92,6 @@ func main() { os.Exit(1) } defer nc.Close() - pub := pubsub.NewPublisher(nc) thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 53e56a92c0..189d9e0059 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -73,7 +73,6 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index bb437b71ab..4b42d89ded 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -136,7 +136,6 @@ func main() { os.Exit(1) } defer nc.Close() - pub := pubsub.NewPublisher(nc) es := mr.NewEventStore(rc, cfg.instance) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index 126014f335..ca19b4ccb4 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -104,7 +104,6 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) ctx := context.Background() diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index 42d36ef386..f13fbee89b 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -82,7 +82,6 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) db := connectToDB(cfg.dbConfig, logger) diff --git a/cmd/twins/main.go b/cmd/twins/main.go index e8d4f570db..b5eb16ddf5 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -120,7 +120,6 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, queue, logger) ncTracer, ncCloser := initJaeger("twins_nats", cfg.jaegerURL, logger) From 49a0b66498f4bdd81424c1e26b5a232bef254382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 20:35:33 +0200 Subject: [PATCH 17/28] Move messaging-related code to messaging package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- Makefile | 2 +- cmd/cassandra-writer/main.go | 2 +- cmd/coap/main.go | 2 +- cmd/http/main.go | 2 +- cmd/influxdb-writer/main.go | 2 +- cmd/lora/main.go | 2 +- cmd/mongodb-writer/main.go | 2 +- cmd/mqtt/main.go | 5 +- cmd/opcua/main.go | 2 +- cmd/postgres-writer/main.go | 2 +- cmd/twins/main.go | 7 +- coap/adapter.go | 11 +- coap/api/logging.go | 4 +- coap/api/metrics.go | 4 +- coap/api/transport.go | 11 +- coap/observer.go | 6 +- go.mod | 2 - http/adapter.go | 9 +- http/api/logging.go | 4 +- http/api/metrics.go | 4 +- http/api/requests.go | 4 +- http/api/transport.go | 11 +- http/mocks/publisher.go | 6 +- lora/adapter.go | 16 ++- messaging/message.pb.go | 125 ++++++++++++++++++ messaging/message.proto | 16 +++ {pubsub => messaging}/nats/doc.go | 0 .../nats/publisher.go | 8 +- {pubsub => messaging}/nats/pubsub.go | 15 ++- messaging/pubsub.go | 26 ++++ mqtt/adapter.go | 18 ++- opcua/gopcua/subscribe.go | 17 ++- transformers/senml/transformer.go | 4 +- transformers/service.go | 4 +- twins/api/logging.go | 4 +- twins/api/metrics.go | 4 +- twins/service.go | 23 ++-- writers/writer.go | 8 +- 38 files changed, 303 insertions(+), 91 deletions(-) create mode 100644 messaging/message.pb.go create mode 100644 messaging/message.proto rename {pubsub => messaging}/nats/doc.go (100%) rename pubsub/nats/pubsliher.go => messaging/nats/publisher.go (72%) rename {pubsub => messaging}/nats/pubsub.go (88%) create mode 100644 messaging/pubsub.go diff --git a/Makefile b/Makefile index 6af40c34f0..fbaabd56c5 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ test: proto: protoc --gofast_out=plugins=grpc:. *.proto + protoc --gofast_out=plugins=grpc:. messaging/*.proto $(SERVICES): $(call compile_service,$(@)) @@ -82,7 +83,6 @@ $(DOCKERS_DEV): $(call make_docker_dev,$(@)) dockers: $(DOCKERS) - dockers_dev: $(DOCKERS_DEV) define docker_push diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 8a6820bfbd..ee014af033 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -17,7 +17,7 @@ import ( "github.com/gocql/gocql" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" diff --git a/cmd/coap/main.go b/cmd/coap/main.go index 67a2fd7731..9750192e14 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -21,7 +21,7 @@ import ( "github.com/mainflux/mainflux/coap" "github.com/mainflux/mainflux/coap/api" logger "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" broker "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" diff --git a/cmd/http/main.go b/cmd/http/main.go index 8fe057448b..c300168e53 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -22,7 +22,7 @@ import ( adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/http/api" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index a8d42a7038..95c7d743af 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -15,7 +15,7 @@ import ( influxdata "github.com/influxdata/influxdb/client/v2" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" diff --git a/cmd/lora/main.go b/cmd/lora/main.go index ca7b3dea60..85539b0ea0 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -19,7 +19,7 @@ import ( "github.com/mainflux/mainflux/lora" "github.com/mainflux/mainflux/lora/api" "github.com/mainflux/mainflux/lora/mqtt" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 189d9e0059..3e457b2e01 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -15,7 +15,7 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index 4b42d89ded..7c6bcf2403 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -15,9 +15,10 @@ import ( "github.com/go-redis/redis" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" + pubsub "github.com/mainflux/mainflux/messaging/nats" mqtt "github.com/mainflux/mainflux/mqtt" mr "github.com/mainflux/mainflux/mqtt/redis" - pubsub "github.com/mainflux/mainflux/pubsub/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" mp "github.com/mainflux/mproxy/pkg/mqtt" "github.com/mainflux/mproxy/pkg/session" @@ -141,7 +142,7 @@ func main() { es := mr.NewEventStore(rc, cfg.instance) // Event handler for MQTT hooks - evt := mqtt.New([]mainflux.Publisher{pub}, cc, es, logger, tracer) + evt := mqtt.New([]messaging.Publisher{pub}, cc, es, logger, tracer) errs := make(chan error, 2) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index ca19b4ccb4..13d1b199e1 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -16,12 +16,12 @@ import ( r "github.com/go-redis/redis" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/opcua" "github.com/mainflux/mainflux/opcua/api" "github.com/mainflux/mainflux/opcua/db" "github.com/mainflux/mainflux/opcua/gopcua" "github.com/mainflux/mainflux/opcua/redis" - pubsub "github.com/mainflux/mainflux/pubsub/nats" "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index f13fbee89b..bf845ad670 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -15,7 +15,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" diff --git a/cmd/twins/main.go b/cmd/twins/main.go index b5eb16ddf5..fd53756f33 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -19,7 +19,8 @@ import ( "github.com/mainflux/mainflux" authapi "github.com/mainflux/mainflux/authn/api/grpc" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + "github.com/mainflux/mainflux/messaging" + pubsub "github.com/mainflux/mainflux/messaging/nats" localusers "github.com/mainflux/mainflux/things/users" "github.com/mainflux/mainflux/twins" "github.com/mainflux/mainflux/twins/api" @@ -237,7 +238,7 @@ func connectToAuth(cfg config, logger logger.Logger) *grpc.ClientConn { return conn } -func newService(ps mainflux.PubSub, ncTracer opentracing.Tracer, chanID string, users mainflux.AuthNServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, logger logger.Logger) twins.Service { +func newService(ps messaging.PubSub, ncTracer opentracing.Tracer, chanID string, users mainflux.AuthNServiceClient, dbTracer opentracing.Tracer, db *mongo.Database, logger logger.Logger) twins.Service { twinRepo := twmongodb.NewTwinRepository(db) stateRepo := twmongodb.NewStateRepository(db) @@ -261,7 +262,7 @@ func newService(ps mainflux.PubSub, ncTracer opentracing.Tracer, chanID string, }, []string{"method"}), ) - err := ps.Subscribe("channels.>", func(msg mainflux.Message) error { + err := ps.Subscribe("channels.>", func(msg messaging.Message) error { if msg.Channel == chanID { return nil } diff --git a/coap/adapter.go b/coap/adapter.go index b97e980ce8..ae626f576b 100644 --- a/coap/adapter.go +++ b/coap/adapter.go @@ -13,6 +13,7 @@ import ( "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" ) const ( @@ -30,7 +31,7 @@ const ( // Service specifies coap service API. type Service interface { // Publish Messssage - Publish(msg mainflux.Message) error + Publish(msg messaging.Message) error // Subscribes to channel with specified id, subtopic and adds subscription to // service map of subscriptions under given ID. @@ -44,14 +45,14 @@ var _ Service = (*adapterService)(nil) type adapterService struct { auth mainflux.ThingsServiceClient - ps mainflux.PubSub + ps messaging.PubSub log logger.Logger obs map[string]*Observer obsLock sync.Mutex } // New instantiates the CoAP adapter implementation. -func New(ps mainflux.PubSub, log logger.Logger, auth mainflux.ThingsServiceClient, responses <-chan string) Service { +func New(ps messaging.PubSub, log logger.Logger, auth mainflux.ThingsServiceClient, responses <-chan string) Service { as := &adapterService{ auth: auth, ps: ps, @@ -107,7 +108,7 @@ func (svc *adapterService) listenResponses(responses <-chan string) { } } -func (svc *adapterService) Publish(msg mainflux.Message) error { +func (svc *adapterService) Publish(msg messaging.Message) error { return svc.ps.Publish(msg.Channel, msg) } @@ -117,7 +118,7 @@ func (svc *adapterService) Subscribe(chanID, subtopic, obsID string, o *Observer subject = fmt.Sprintf("%s.%s", chanID, subtopic) } - err := svc.ps.Subscribe(subject, func(msg mainflux.Message) error { + err := svc.ps.Subscribe(subject, func(msg messaging.Message) error { o.Messages <- msg return nil }) diff --git a/coap/api/logging.go b/coap/api/logging.go index a070cd26e9..9d846a7a99 100644 --- a/coap/api/logging.go +++ b/coap/api/logging.go @@ -9,9 +9,9 @@ import ( "fmt" "time" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/coap" log "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" ) var _ coap.Service = (*loggingMiddleware)(nil) @@ -26,7 +26,7 @@ func LoggingMiddleware(svc coap.Service, logger log.Logger) coap.Service { return &loggingMiddleware{logger, svc} } -func (lm *loggingMiddleware) Publish(msg mainflux.Message) (err error) { +func (lm *loggingMiddleware) Publish(msg messaging.Message) (err error) { defer func(begin time.Time) { destChannel := msg.Channel if msg.Subtopic != "" { diff --git a/coap/api/metrics.go b/coap/api/metrics.go index 7b033e11f0..7d30ec9eaf 100644 --- a/coap/api/metrics.go +++ b/coap/api/metrics.go @@ -9,8 +9,8 @@ import ( "time" "github.com/go-kit/kit/metrics" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/coap" + "github.com/mainflux/mainflux/messaging" ) var _ coap.Service = (*metricsMiddleware)(nil) @@ -30,7 +30,7 @@ func MetricsMiddleware(svc coap.Service, counter metrics.Counter, latency metric } } -func (mm *metricsMiddleware) Publish(msg mainflux.Message) error { +func (mm *metricsMiddleware) Publish(msg messaging.Message) error { defer func(begin time.Time) { mm.counter.With("method", "publish").Add(1) mm.latency.With("method", "publish").Observe(time.Since(begin).Seconds()) diff --git a/coap/api/transport.go b/coap/api/transport.go index 103374b5d6..f5eb2e231a 100644 --- a/coap/api/transport.go +++ b/coap/api/transport.go @@ -18,9 +18,11 @@ import ( gocoap "github.com/dustin/go-coap" "github.com/go-zoo/bone" + "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/coap" log "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -223,13 +225,18 @@ func receive(svc coap.Service, msg *gocoap.Message) *gocoap.Message { return res } - m := mainflux.Message{ + occured, err := ptypes.TimestampProto(time.Now()) + if err != nil { + return nil + } + + m := messaging.Message{ Channel: chanID, Subtopic: subtopic, Publisher: publisher, Protocol: protocol, Payload: msg.Payload, - Occurred: time.Now().UnixNano(), + Occurred: occured, } if err := svc.Publish(m); err != nil { diff --git a/coap/observer.go b/coap/observer.go index ad076fe6a4..3788d96838 100644 --- a/coap/observer.go +++ b/coap/observer.go @@ -6,7 +6,7 @@ package coap import ( "sync" - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/messaging" ) // Observer is used to handle CoAP subscription. @@ -25,7 +25,7 @@ type Observer struct { expiredLock, msgIDLock sync.Mutex // Messages is used to receive messages from NATS. - Messages chan mainflux.Message + Messages chan messaging.Message // Cancel channel is used to cancel observing resource. // Cancel channel should not be used to send or receive any @@ -36,7 +36,7 @@ type Observer struct { // NewObserver instantiates a new Observer. func NewObserver() *Observer { return &Observer{ - Messages: make(chan mainflux.Message), + Messages: make(chan messaging.Message), Cancel: make(chan bool), } } diff --git a/go.mod b/go.mod index 981e7c45ad..4ca03931bb 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/lib/pq v1.0.0 github.com/mainflux/mproxy v0.1.8 github.com/mainflux/senml v1.0.1 - github.com/nats-io/go-nats v1.6.0 // indirect github.com/nats-io/nats.go v1.9.1 github.com/opentracing/opentracing-go v1.1.0 github.com/prometheus/client_golang v0.9.3 @@ -36,7 +35,6 @@ require ( golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b gonum.org/v1/gonum v0.0.0-20190808205415-ced62fe5104b - google.golang.org/genproto v0.0.0-20200225123651-fc8f55426688 // indirect google.golang.org/grpc v1.27.1 gopkg.in/ory/dockertest.v3 v3.3.5 ) diff --git a/http/adapter.go b/http/adapter.go index a1d7b4f4ba..e51a5727f9 100644 --- a/http/adapter.go +++ b/http/adapter.go @@ -9,30 +9,31 @@ import ( "context" "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/messaging" ) // Service specifies coap service API. type Service interface { // Publish Messssage - Publish(ctx context.Context, token string, msg mainflux.Message) error + Publish(ctx context.Context, token string, msg messaging.Message) error } var _ Service = (*adapterService)(nil) type adapterService struct { - publisher mainflux.Publisher + publisher messaging.Publisher things mainflux.ThingsServiceClient } // New instantiates the HTTP adapter implementation. -func New(publisher mainflux.Publisher, things mainflux.ThingsServiceClient) Service { +func New(publisher messaging.Publisher, things mainflux.ThingsServiceClient) Service { return &adapterService{ publisher: publisher, things: things, } } -func (as *adapterService) Publish(ctx context.Context, token string, msg mainflux.Message) error { +func (as *adapterService) Publish(ctx context.Context, token string, msg messaging.Message) error { ar := &mainflux.AccessByKeyReq{ Token: token, ChanID: msg.Channel, diff --git a/http/api/logging.go b/http/api/logging.go index 8a41d43929..00eb3c2abe 100644 --- a/http/api/logging.go +++ b/http/api/logging.go @@ -10,9 +10,9 @@ import ( "fmt" "time" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/http" log "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" ) var _ http.Service = (*loggingMiddleware)(nil) @@ -27,7 +27,7 @@ func LoggingMiddleware(svc http.Service, logger log.Logger) http.Service { return &loggingMiddleware{logger, svc} } -func (lm *loggingMiddleware) Publish(ctx context.Context, token string, msg mainflux.Message) (err error) { +func (lm *loggingMiddleware) Publish(ctx context.Context, token string, msg messaging.Message) (err error) { defer func(begin time.Time) { destChannel := msg.Channel if msg.Subtopic != "" { diff --git a/http/api/metrics.go b/http/api/metrics.go index d7b9ffb975..c91f354e64 100644 --- a/http/api/metrics.go +++ b/http/api/metrics.go @@ -10,8 +10,8 @@ import ( "time" "github.com/go-kit/kit/metrics" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/http" + "github.com/mainflux/mainflux/messaging" ) var _ http.Service = (*metricsMiddleware)(nil) @@ -31,7 +31,7 @@ func MetricsMiddleware(svc http.Service, counter metrics.Counter, latency metric } } -func (mm *metricsMiddleware) Publish(ctx context.Context, token string, msg mainflux.Message) error { +func (mm *metricsMiddleware) Publish(ctx context.Context, token string, msg messaging.Message) error { defer func(begin time.Time) { mm.counter.With("method", "publish").Add(1) mm.latency.With("method", "publish").Observe(time.Since(begin).Seconds()) diff --git a/http/api/requests.go b/http/api/requests.go index 4c7afc412f..77f72bba6e 100644 --- a/http/api/requests.go +++ b/http/api/requests.go @@ -4,10 +4,10 @@ package api import ( - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/messaging" ) type publishReq struct { - msg mainflux.Message + msg messaging.Message token string } diff --git a/http/api/transport.go b/http/api/transport.go index 72d72e22a1..e4517c9035 100644 --- a/http/api/transport.go +++ b/http/api/transport.go @@ -17,8 +17,10 @@ import ( kitot "github.com/go-kit/kit/tracing/opentracing" kithttp "github.com/go-kit/kit/transport/http" "github.com/go-zoo/bone" + "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux" adapter "github.com/mainflux/mainflux/http" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/things" opentracing "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -108,12 +110,17 @@ func decodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { return nil, err } - msg := mainflux.Message{ + occurred, err := ptypes.TimestampProto(time.Now()) + if err != nil { + return nil, err + } + + msg := messaging.Message{ Protocol: protocol, Channel: chanID, Subtopic: subtopic, Payload: payload, - Occurred: time.Now().UnixNano(), + Occurred: occurred, } req := publishReq{ diff --git a/http/mocks/publisher.go b/http/mocks/publisher.go index 7d08dac016..dddb6f008a 100644 --- a/http/mocks/publisher.go +++ b/http/mocks/publisher.go @@ -4,16 +4,16 @@ package mocks import ( - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/messaging" ) type mockPublisher struct{} // NewPublisher returns mock message publisher. -func NewPublisher() mainflux.Publisher { +func NewPublisher() messaging.Publisher { return mockPublisher{} } -func (pub mockPublisher) Publish(topic string, msg mainflux.Message) error { +func (pub mockPublisher) Publish(topic string, msg messaging.Message) error { return nil } diff --git a/lora/adapter.go b/lora/adapter.go index 0fad3417d7..28bd4bc79c 100644 --- a/lora/adapter.go +++ b/lora/adapter.go @@ -7,7 +7,8 @@ import ( "errors" "time" - "github.com/mainflux/mainflux" + "github.com/golang/protobuf/ptypes" + "github.com/mainflux/mainflux/messaging" ) const ( @@ -55,13 +56,13 @@ type Service interface { var _ Service = (*adapterService)(nil) type adapterService struct { - publisher mainflux.Publisher + publisher messaging.Publisher thingsRM RouteMapRepository channelsRM RouteMapRepository } // New instantiates the LoRa adapter implementation. -func New(publisher mainflux.Publisher, thingsRM, channelsRM RouteMapRepository) Service { +func New(publisher messaging.Publisher, thingsRM, channelsRM RouteMapRepository) Service { return &adapterService{ publisher: publisher, thingsRM: thingsRM, @@ -100,13 +101,18 @@ func (as *adapterService) Publish(ctx context.Context, token string, m Message) payload = []byte(jo) } + occured, err := ptypes.TimestampProto(time.Now()) + if err != nil { + return nil + } + // Publish on Mainflux NATS broker - msg := mainflux.Message{ + msg := messaging.Message{ Publisher: thing, Protocol: protocol, Channel: channel, Payload: payload, - Occurred: time.Now().UnixNano(), + Occurred: occured, } return as.publisher.Publish(msg.Channel, msg) diff --git a/messaging/message.pb.go b/messaging/message.pb.go new file mode 100644 index 0000000000..ca92501b90 --- /dev/null +++ b/messaging/message.pb.go @@ -0,0 +1,125 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: broker/message.proto + +package messaging + +import ( + fmt "fmt" + math "math" + + proto "github.com/gogo/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// Message represents a message emitted by the Mainflux adapters layer. +type Message struct { + Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` + Subtopic string `protobuf:"bytes,2,opt,name=subtopic,proto3" json:"subtopic,omitempty"` + Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` + Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` + Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` + Occurred *timestamp.Timestamp `protobuf:"bytes,6,opt,name=created,proto3" json:"created,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} +func (*Message) Descriptor() ([]byte, []int) { + return fileDescriptor_6357da820a7eacc2, []int{0} +} +func (m *Message) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Message.Unmarshal(m, b) +} +func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Message.Marshal(b, m, deterministic) +} +func (m *Message) XXX_Merge(src proto.Message) { + xxx_messageInfo_Message.Merge(m, src) +} +func (m *Message) XXX_Size() int { + return xxx_messageInfo_Message.Size(m) +} +func (m *Message) XXX_DiscardUnknown() { + xxx_messageInfo_Message.DiscardUnknown(m) +} + +var xxx_messageInfo_Message proto.InternalMessageInfo + +func (m *Message) GetChannel() string { + if m != nil { + return m.Channel + } + return "" +} + +func (m *Message) GetSubtopic() string { + if m != nil { + return m.Subtopic + } + return "" +} + +func (m *Message) GetPublisher() string { + if m != nil { + return m.Publisher + } + return "" +} + +func (m *Message) GetProtocol() string { + if m != nil { + return m.Protocol + } + return "" +} + +func (m *Message) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func (m *Message) GetCreated() *timestamp.Timestamp { + if m != nil { + return m.Occurred + } + return nil +} + +func init() { + proto.RegisterType((*Message)(nil), "broker.Message") +} + +func init() { proto.RegisterFile("broker/message.proto", fileDescriptor_6357da820a7eacc2) } + +var fileDescriptor_6357da820a7eacc2 = []byte{ + // 202 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x8f, 0xcb, 0x6e, 0x84, 0x20, + 0x18, 0x85, 0x43, 0x2f, 0x5a, 0x69, 0x57, 0xa4, 0x0b, 0x62, 0x9a, 0xd4, 0x74, 0xe5, 0x0a, 0x93, + 0xb6, 0xaf, 0x31, 0x1b, 0x33, 0x2f, 0x00, 0xf8, 0x8f, 0x9a, 0x41, 0x7f, 0x02, 0xb8, 0x98, 0x47, + 0x9c, 0xb7, 0x9a, 0x08, 0x61, 0x66, 0xf9, 0x9d, 0x4b, 0x4e, 0x0e, 0xfd, 0x54, 0x0e, 0xcf, 0xe0, + 0xba, 0x05, 0xbc, 0x97, 0x23, 0x08, 0xeb, 0x30, 0x20, 0x2b, 0x92, 0x5a, 0x7f, 0x8f, 0x88, 0xa3, + 0x81, 0x2e, 0xaa, 0x6a, 0x3b, 0x75, 0x61, 0x5e, 0xc0, 0x07, 0xb9, 0xd8, 0x14, 0xfc, 0xb9, 0x12, + 0x5a, 0x1e, 0x52, 0x95, 0x71, 0x5a, 0xea, 0x49, 0xae, 0x2b, 0x18, 0x4e, 0x1a, 0xd2, 0x56, 0x7d, + 0x46, 0x56, 0xd3, 0x37, 0xbf, 0xa9, 0x80, 0x76, 0xd6, 0xfc, 0x29, 0x5a, 0x77, 0x66, 0x5f, 0xb4, + 0xb2, 0x9b, 0x32, 0xb3, 0x9f, 0xc0, 0xf1, 0xe7, 0x68, 0x3e, 0x84, 0xbd, 0x19, 0x87, 0x34, 0x1a, + 0xfe, 0x92, 0x9a, 0x99, 0xf7, 0x3d, 0x2b, 0x2f, 0x06, 0xe5, 0xc0, 0x5f, 0x1b, 0xd2, 0x7e, 0xf4, + 0x19, 0xd9, 0x3f, 0x2d, 0xb5, 0x03, 0x19, 0x60, 0xe0, 0x45, 0x43, 0xda, 0xf7, 0xdf, 0x5a, 0xa4, + 0x23, 0x22, 0x1f, 0x11, 0xc7, 0x7c, 0xa4, 0xcf, 0x51, 0x55, 0x44, 0xf3, 0xef, 0x16, 0x00, 0x00, + 0xff, 0xff, 0x0b, 0xd8, 0x04, 0x71, 0x13, 0x01, 0x00, 0x00, +} diff --git a/messaging/message.proto b/messaging/message.proto new file mode 100644 index 0000000000..4f44023372 --- /dev/null +++ b/messaging/message.proto @@ -0,0 +1,16 @@ +// Copyright (c) Mainflux +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; +package messaging; +import "google/protobuf/timestamp.proto"; + +// Message represents a message emitted by the Mainflux adapters layer. +message Message { + string channel = 1; + string subtopic = 2; + string publisher = 3; + string protocol = 4; + bytes payload = 5; + google.protobuf.Timestamp occurred = 6; +} diff --git a/pubsub/nats/doc.go b/messaging/nats/doc.go similarity index 100% rename from pubsub/nats/doc.go rename to messaging/nats/doc.go diff --git a/pubsub/nats/pubsliher.go b/messaging/nats/publisher.go similarity index 72% rename from pubsub/nats/pubsliher.go rename to messaging/nats/publisher.go index 33c19f60e6..56382b3afd 100644 --- a/pubsub/nats/pubsliher.go +++ b/messaging/nats/publisher.go @@ -7,24 +7,24 @@ import ( "fmt" "github.com/gogo/protobuf/proto" - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/messaging" broker "github.com/nats-io/nats.go" ) -var _ mainflux.PubSub = (*pubsub)(nil) +var _ messaging.Publisher = (*pubsliher)(nil) type pubsliher struct { conn *broker.Conn } // NewPublisher returns NATS message Publisher. -func NewPublisher(conn *broker.Conn) mainflux.Publisher { +func NewPublisher(conn *broker.Conn) messaging.Publisher { return &pubsub{ conn: conn, } } -func (pub *pubsliher) Publish(topic string, msg mainflux.Message) error { +func (pub *pubsliher) Publish(topic string, msg messaging.Message) error { data, err := proto.Marshal(&msg) if err != nil { return err diff --git a/pubsub/nats/pubsub.go b/messaging/nats/pubsub.go similarity index 88% rename from pubsub/nats/pubsub.go rename to messaging/nats/pubsub.go index b06718fd6c..1b7e0e5b13 100644 --- a/pubsub/nats/pubsub.go +++ b/messaging/nats/pubsub.go @@ -9,8 +9,9 @@ import ( "sync" "github.com/gogo/protobuf/proto" - "github.com/mainflux/mainflux" + log "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" broker "github.com/nats-io/nats.go" ) @@ -25,7 +26,7 @@ var ( errEmptyTopic = errors.New("empty topic") ) -var _ mainflux.PubSub = (*pubsub)(nil) +var _ messaging.PubSub = (*pubsub)(nil) type pubsub struct { conn *broker.Conn @@ -42,7 +43,7 @@ type pubsub struct { // from ordinary subscribe. For more information, please take a look // here: https://docs.nats.io/developing-with-nats/receiving/queues. // If the queue is empty, Subscribe will be used. -func NewPubSub(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubSub { +func NewPubSub(conn *broker.Conn, queue string, logger log.Logger) messaging.PubSub { return &pubsub{ conn: conn, queue: queue, @@ -51,7 +52,7 @@ func NewPubSub(conn *broker.Conn, queue string, logger log.Logger) mainflux.PubS } } -func (n *pubsub) Publish(topic string, msg mainflux.Message) error { +func (n *pubsub) Publish(topic string, msg messaging.Message) error { data, err := proto.Marshal(&msg) if err != nil { return err @@ -68,7 +69,7 @@ func (n *pubsub) Publish(topic string, msg mainflux.Message) error { return nil } -func (n *pubsub) Subscribe(topic string, handler mainflux.MessageHandler) error { +func (n *pubsub) Subscribe(topic string, handler messaging.MessageHandler) error { if topic == "" { return errEmptyTopic } @@ -116,9 +117,9 @@ func (n *pubsub) Unsubscribe(topic string) error { return nil } -func (n *pubsub) natsHandler(h mainflux.MessageHandler) broker.MsgHandler { +func (n *pubsub) natsHandler(h messaging.MessageHandler) broker.MsgHandler { return func(m *broker.Msg) { - var msg mainflux.Message + var msg messaging.Message if err := proto.Unmarshal(m.Data, &msg); err != nil { n.logger.Warn(fmt.Sprintf("Failed to unmarshal received message: %s", err)) return diff --git a/messaging/pubsub.go b/messaging/pubsub.go new file mode 100644 index 0000000000..fac02ea3b2 --- /dev/null +++ b/messaging/pubsub.go @@ -0,0 +1,26 @@ +package messaging + +// Publisher specifies message publishing API. +type Publisher interface { + // Publishes message to the stream. + Publish(topic string, msg Message) error +} + +// MessageHandler represents Message handler for Subscriber. +type MessageHandler func(msg Message) error + +// Subscriber specifies message subscription API. +type Subscriber interface { + // Subscribe subscribes to the message stream and consumes messages. + Subscribe(topic string, handler MessageHandler) error + + // Unsubscribe unsubscribes from the message stream and + // stops consuming messages. + Unsubscribe(topic string) error +} + +// PubSub represents aggregation interface for publisher and subscriber. +type PubSub interface { + Publisher + Subscriber +} diff --git a/mqtt/adapter.go b/mqtt/adapter.go index 4c8c31dbea..bc2e267f77 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -11,8 +11,10 @@ import ( "strings" "time" + "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/mqtt/redis" "github.com/mainflux/mproxy/pkg/session" opentracing "github.com/opentracing/opentracing-go" @@ -36,7 +38,7 @@ var ( // Event implements events.Event interface type handler struct { - publishers []mainflux.Publisher + publishers []messaging.Publisher tc mainflux.ThingsServiceClient tracer opentracing.Tracer logger logger.Logger @@ -44,7 +46,7 @@ type handler struct { } // New creates new Event entity -func New(publishers []mainflux.Publisher, tc mainflux.ThingsServiceClient, es redis.EventStore, +func New(publishers []messaging.Publisher, tc mainflux.ThingsServiceClient, es redis.EventStore, logger logger.Logger, tracer opentracing.Tracer) session.Handler { return &handler{ tc: tc, @@ -145,17 +147,23 @@ func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { subtopic, err := parseSubtopic(subtopic) if err != nil { - h.logger.Info("Error in mqtt publish: " + err.Error()) + h.logger.Info("Error parsing subtopic: " + err.Error()) return } - msg := mainflux.Message{ + occured, err := ptypes.TimestampProto(time.Now()) + if err != nil { + h.logger.Info("Error creating message timestamp: " + err.Error()) + return + } + + msg := messaging.Message{ Protocol: protocol, Channel: chanID, Subtopic: subtopic, Publisher: c.Username, Payload: *payload, - Occurred: time.Now().UnixNano(), + Occurred: occured, } for _, pub := range h.publishers { diff --git a/opcua/gopcua/subscribe.go b/opcua/gopcua/subscribe.go index 1d7adbb689..ebc15161a4 100644 --- a/opcua/gopcua/subscribe.go +++ b/opcua/gopcua/subscribe.go @@ -9,11 +9,12 @@ import ( "strconv" "time" + "github.com/golang/protobuf/ptypes" opcuaGopcua "github.com/gopcua/opcua" uaGopcua "github.com/gopcua/opcua/ua" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/opcua" ) @@ -40,7 +41,7 @@ var _ opcua.Subscriber = (*client)(nil) type client struct { ctx context.Context - publisher mainflux.Publisher + publisher messaging.Publisher thingsRM opcua.RouteMapRepository channelsRM opcua.RouteMapRepository connectRM opcua.RouteMapRepository @@ -57,7 +58,7 @@ type message struct { } // NewSubscriber returns new OPC-UA client instance. -func NewSubscriber(ctx context.Context, publisher mainflux.Publisher, thingsRM, channelsRM, connectRM opcua.RouteMapRepository, log logger.Logger) opcua.Subscriber { +func NewSubscriber(ctx context.Context, publisher messaging.Publisher, thingsRM, channelsRM, connectRM opcua.RouteMapRepository, log logger.Logger) opcua.Subscriber { return client{ ctx: ctx, publisher: publisher, @@ -218,6 +219,11 @@ func (c client) publish(token string, m message) error { return errNotFoundNodeID } + occured, err := ptypes.TimestampProto(time.Now()) + if err != nil { + return nil + } + // Check connection between ServerURI and NodeID cKey := fmt.Sprintf("%s:%s", chanID, thingID) if _, err := c.connectRM.Get(cKey); err != nil { @@ -227,13 +233,14 @@ func (c client) publish(token string, m message) error { // Publish on Mainflux NATS broker SenML := fmt.Sprintf(`[{"n":"%s", "t": %d, "%s":%v}]`, m.Type, m.Time, m.DataKey, m.Data) payload := []byte(SenML) - msg := mainflux.Message{ + + msg := messaging.Message{ Publisher: thingID, Protocol: protocol, Channel: chanID, Payload: payload, Subtopic: m.NodeID, - Occurred: time.Now().UnixNano(), + Occurred: occured, } if err := c.publisher.Publish(msg.Channel, msg); err != nil { diff --git a/transformers/senml/transformer.go b/transformers/senml/transformer.go index a29351f3f7..c0224a7cb2 100644 --- a/transformers/senml/transformer.go +++ b/transformers/senml/transformer.go @@ -6,8 +6,8 @@ package senml import ( "time" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/transformers" "github.com/mainflux/senml" ) @@ -45,7 +45,7 @@ func New(contentFormat string) transformers.Transformer { } } -func (t transformer) Transform(msg mainflux.Message) (interface{}, error) { +func (t transformer) Transform(msg messaging.Message) (interface{}, error) { raw, err := senml.Decode(msg.Payload, t.format) if err != nil { return nil, errors.Wrap(errDecode, err) diff --git a/transformers/service.go b/transformers/service.go index ec0cb9a0ba..a2789e9a3a 100644 --- a/transformers/service.go +++ b/transformers/service.go @@ -3,10 +3,10 @@ package transformers -import "github.com/mainflux/mainflux" +import "github.com/mainflux/mainflux/messaging" // Transformer specifies API form Message transformer. type Transformer interface { // Transform Mainflux message to any other format. - Transform(msg mainflux.Message) (interface{}, error) + Transform(msg messaging.Message) (interface{}, error) } diff --git a/twins/api/logging.go b/twins/api/logging.go index 1d070c861f..1bad133b58 100644 --- a/twins/api/logging.go +++ b/twins/api/logging.go @@ -10,8 +10,8 @@ import ( "fmt" "time" - "github.com/mainflux/mainflux" log "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/twins" ) @@ -79,7 +79,7 @@ func (lm *loggingMiddleware) ListTwins(ctx context.Context, token string, offset return lm.svc.ListTwins(ctx, token, offset, limit, name, metadata) } -func (lm *loggingMiddleware) SaveStates(msg *mainflux.Message) (err error) { +func (lm *loggingMiddleware) SaveStates(msg *messaging.Message) (err error) { defer func(begin time.Time) { message := fmt.Sprintf("Method save_states took %s to complete", time.Since(begin)) if err != nil { diff --git a/twins/api/metrics.go b/twins/api/metrics.go index c88a7be8a6..35655808de 100644 --- a/twins/api/metrics.go +++ b/twins/api/metrics.go @@ -10,7 +10,7 @@ import ( "time" "github.com/go-kit/kit/metrics" - "github.com/mainflux/mainflux" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/twins" ) @@ -68,7 +68,7 @@ func (ms *metricsMiddleware) ListTwins(ctx context.Context, token string, offset return ms.svc.ListTwins(ctx, token, offset, limit, name, metadata) } -func (ms *metricsMiddleware) SaveStates(msg *mainflux.Message) error { +func (ms *metricsMiddleware) SaveStates(msg *messaging.Message) error { defer func(begin time.Time) { ms.counter.With("method", "save_states").Add(1) ms.latency.With("method", "save_states").Observe(time.Since(begin).Seconds()) diff --git a/twins/service.go b/twins/service.go index cbd858772b..8c82aa120b 100644 --- a/twins/service.go +++ b/twins/service.go @@ -10,8 +10,10 @@ import ( "math" "time" + "github.com/golang/protobuf/ptypes" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux" "github.com/mainflux/senml" @@ -69,7 +71,7 @@ type Service interface { ListStates(ctx context.Context, token string, offset uint64, limit uint64, id string) (StatesPage, error) // SaveStates persists states into database - SaveStates(msg *mainflux.Message) error + SaveStates(msg *messaging.Message) error } const ( @@ -94,7 +96,7 @@ var crudOp = map[string]string{ } type twinsService struct { - publisher mainflux.Publisher + publisher messaging.Publisher auth mainflux.AuthNServiceClient twins TwinRepository states StateRepository @@ -106,7 +108,7 @@ type twinsService struct { var _ Service = (*twinsService)(nil) // New instantiates the twins service implementation. -func New(publisher mainflux.Publisher, auth mainflux.AuthNServiceClient, twins TwinRepository, sr StateRepository, idp IdentityProvider, chann string, logger logger.Logger) Service { +func New(publisher messaging.Publisher, auth mainflux.AuthNServiceClient, twins TwinRepository, sr StateRepository, idp IdentityProvider, chann string, logger logger.Logger) Service { return &twinsService{ publisher: publisher, auth: auth, @@ -278,7 +280,7 @@ func (ts *twinsService) ListStates(ctx context.Context, token string, offset uin return ts.states.RetrieveAll(ctx, offset, limit, id) } -func (ts *twinsService) SaveStates(msg *mainflux.Message) error { +func (ts *twinsService) SaveStates(msg *messaging.Message) error { ids, err := ts.twins.RetrieveByAttribute(context.TODO(), msg.Channel, msg.Subtopic) if err != nil { return err @@ -293,7 +295,7 @@ func (ts *twinsService) SaveStates(msg *mainflux.Message) error { return nil } -func (ts *twinsService) saveState(msg *mainflux.Message, id string) error { +func (ts *twinsService) saveState(msg *messaging.Message, id string) error { var b []byte var err error defer ts.publish(&id, &err, crudOp["stateSucc"], crudOp["stateFail"], &b) @@ -335,7 +337,7 @@ func (ts *twinsService) saveState(msg *mainflux.Message, id string) error { return nil } -func prepareState(st *State, tw *Twin, rec senml.Record, msg *mainflux.Message) int { +func prepareState(st *State, tw *Twin, rec senml.Record, msg *messaging.Message) int { def := tw.Definitions[len(tw.Definitions)-1] st.TwinID = tw.ID st.Definition = def.ID @@ -426,12 +428,17 @@ func (ts *twinsService) publish(twinID *string, err *error, succOp, failOp strin pl = []byte(fmt.Sprintf("{\"deleted\":\"%s\"}", *twinID)) } - msg := mainflux.Message{ + occured, timeErr := ptypes.TimestampProto(time.Now()) + if timeErr != nil { + return + } + + msg := messaging.Message{ Channel: ts.channelID, Subtopic: op, Payload: pl, Publisher: publisher, - Occurred: time.Now().UnixNano(), + Occurred: occured, } if err := ts.publisher.Publish(msg.Channel, msg); err != nil { diff --git a/writers/writer.go b/writers/writer.go index 69c0574d1e..b364118cde 100644 --- a/writers/writer.go +++ b/writers/writer.go @@ -8,10 +8,10 @@ import ( "io/ioutil" "github.com/BurntSushi/toml" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/pubsub/nats" + "github.com/mainflux/mainflux/messaging" + pubsub "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers" "github.com/mainflux/mainflux/transformers/senml" ) @@ -31,7 +31,7 @@ type consumer struct { // Start method starts consuming messages received from NATS. // This method transforms messages to SenML format before // using MessageRepository to store them. -func Start(sub mainflux.Subscriber, repo MessageRepository, transformer transformers.Transformer, queue string, subjectsCfgPath string, logger logger.Logger) error { +func Start(sub messaging.Subscriber, repo MessageRepository, transformer transformers.Transformer, queue string, subjectsCfgPath string, logger logger.Logger) error { c := consumer{ repo: repo, transformer: transformer, @@ -51,7 +51,7 @@ func Start(sub mainflux.Subscriber, repo MessageRepository, transformer transfor return nil } -func (c *consumer) handler(msg mainflux.Message) error { +func (c *consumer) handler(msg messaging.Message) error { t, err := c.transformer.Transform(msg) if err != nil { return err From 1c50f8c6f8cdd4cda45899928d1f213c3040122a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 20:45:30 +0200 Subject: [PATCH 18/28] Fix Twins mocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- twins/mocks/messages.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/twins/mocks/messages.go b/twins/mocks/messages.go index bda1cd13c9..43611b33c7 100644 --- a/twins/mocks/messages.go +++ b/twins/mocks/messages.go @@ -4,24 +4,24 @@ package mocks import ( - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" + "github.com/mainflux/mainflux/messaging" ) -var _ mainflux.Publisher = (*mockBroker)(nil) +var _ messaging.Publisher = (*mockBroker)(nil) type mockBroker struct { subscriptions map[string]string } // New returns mock message publisher. -func New(sub map[string]string) mainflux.Publisher { +func New(sub map[string]string) messaging.Publisher { return &mockBroker{ subscriptions: sub, } } -func (mb mockBroker) Publish(topic string, msg mainflux.Message) error { +func (mb mockBroker) Publish(topic string, msg messaging.Message) error { if len(msg.Payload) == 0 { return errors.New("failed to publish") } From bd296d1eff44898419ea58904b13fafb31bc65a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 21:14:00 +0200 Subject: [PATCH 19/28] Change Occurred back to Created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- coap/api/transport.go | 2 +- http/api/transport.go | 4 ++-- lora/adapter.go | 4 ++-- messaging/message.pb.go | 6 +++--- messaging/message.proto | 2 +- mqtt/adapter.go | 4 ++-- opcua/gopcua/subscribe.go | 4 ++-- twins/service.go | 4 ++-- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/coap/api/transport.go b/coap/api/transport.go index f5eb2e231a..017487f691 100644 --- a/coap/api/transport.go +++ b/coap/api/transport.go @@ -236,7 +236,7 @@ func receive(svc coap.Service, msg *gocoap.Message) *gocoap.Message { Publisher: publisher, Protocol: protocol, Payload: msg.Payload, - Occurred: occured, + Created: occured, } if err := svc.Publish(m); err != nil { diff --git a/http/api/transport.go b/http/api/transport.go index e4517c9035..cc6d6b8469 100644 --- a/http/api/transport.go +++ b/http/api/transport.go @@ -110,7 +110,7 @@ func decodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { return nil, err } - occurred, err := ptypes.TimestampProto(time.Now()) + created, err := ptypes.TimestampProto(time.Now()) if err != nil { return nil, err } @@ -120,7 +120,7 @@ func decodeRequest(ctx context.Context, r *http.Request) (interface{}, error) { Channel: chanID, Subtopic: subtopic, Payload: payload, - Occurred: occurred, + Created: created, } req := publishReq{ diff --git a/lora/adapter.go b/lora/adapter.go index 28bd4bc79c..43e636ec78 100644 --- a/lora/adapter.go +++ b/lora/adapter.go @@ -101,7 +101,7 @@ func (as *adapterService) Publish(ctx context.Context, token string, m Message) payload = []byte(jo) } - occured, err := ptypes.TimestampProto(time.Now()) + created, err := ptypes.TimestampProto(time.Now()) if err != nil { return nil } @@ -112,7 +112,7 @@ func (as *adapterService) Publish(ctx context.Context, token string, m Message) Protocol: protocol, Channel: channel, Payload: payload, - Occurred: occured, + Created: created, } return as.publisher.Publish(msg.Channel, msg) diff --git a/messaging/message.pb.go b/messaging/message.pb.go index ca92501b90..4995187819 100644 --- a/messaging/message.pb.go +++ b/messaging/message.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: broker/message.proto +// source: messaging/message.proto package messaging @@ -29,7 +29,7 @@ type Message struct { Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` - Occurred *timestamp.Timestamp `protobuf:"bytes,6,opt,name=created,proto3" json:"created,omitempty"` + Created *timestamp.Timestamp `protobuf:"bytes,6,opt,name=created,proto3" json:"created,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -96,7 +96,7 @@ func (m *Message) GetPayload() []byte { func (m *Message) GetCreated() *timestamp.Timestamp { if m != nil { - return m.Occurred + return m.Created } return nil } diff --git a/messaging/message.proto b/messaging/message.proto index 4f44023372..936111aa34 100644 --- a/messaging/message.proto +++ b/messaging/message.proto @@ -12,5 +12,5 @@ message Message { string publisher = 3; string protocol = 4; bytes payload = 5; - google.protobuf.Timestamp occurred = 6; + google.protobuf.Timestamp created = 6; } diff --git a/mqtt/adapter.go b/mqtt/adapter.go index bc2e267f77..60e6e36329 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -151,7 +151,7 @@ func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { return } - occured, err := ptypes.TimestampProto(time.Now()) + created, err := ptypes.TimestampProto(time.Now()) if err != nil { h.logger.Info("Error creating message timestamp: " + err.Error()) return @@ -163,7 +163,7 @@ func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { Subtopic: subtopic, Publisher: c.Username, Payload: *payload, - Occurred: occured, + Created: created, } for _, pub := range h.publishers { diff --git a/opcua/gopcua/subscribe.go b/opcua/gopcua/subscribe.go index ebc15161a4..5120af07b7 100644 --- a/opcua/gopcua/subscribe.go +++ b/opcua/gopcua/subscribe.go @@ -219,7 +219,7 @@ func (c client) publish(token string, m message) error { return errNotFoundNodeID } - occured, err := ptypes.TimestampProto(time.Now()) + created, err := ptypes.TimestampProto(time.Now()) if err != nil { return nil } @@ -240,7 +240,7 @@ func (c client) publish(token string, m message) error { Channel: chanID, Payload: payload, Subtopic: m.NodeID, - Occurred: occured, + Created: created, } if err := c.publisher.Publish(msg.Channel, msg); err != nil { diff --git a/twins/service.go b/twins/service.go index 8c82aa120b..1b62581787 100644 --- a/twins/service.go +++ b/twins/service.go @@ -428,7 +428,7 @@ func (ts *twinsService) publish(twinID *string, err *error, succOp, failOp strin pl = []byte(fmt.Sprintf("{\"deleted\":\"%s\"}", *twinID)) } - occured, timeErr := ptypes.TimestampProto(time.Now()) + created, timeErr := ptypes.TimestampProto(time.Now()) if timeErr != nil { return } @@ -438,7 +438,7 @@ func (ts *twinsService) publish(twinID *string, err *error, succOp, failOp strin Subtopic: op, Payload: pl, Publisher: publisher, - Occurred: occured, + Created: created, } if err := ts.publisher.Publish(msg.Channel, msg); err != nil { From b19d0f342a66ffc851f8722794e45f8a1c9f46ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 21:20:24 +0200 Subject: [PATCH 20/28] Fix tranformer test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- transformers/senml/transformer_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/transformers/senml/transformer_test.go b/transformers/senml/transformer_test.go index 73ac1f8993..1526bf40a0 100644 --- a/transformers/senml/transformer_test.go +++ b/transformers/senml/transformer_test.go @@ -8,8 +8,8 @@ import ( "fmt" "testing" - "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/errors" + "github.com/mainflux/mainflux/messaging" "github.com/mainflux/mainflux/transformers/senml" mfsenml "github.com/mainflux/senml" "github.com/stretchr/testify/assert" @@ -24,7 +24,7 @@ func TestTransformJSON(t *testing.T) { require.Nil(t, err, "Decoding JSON expected to succeed") tr := senml.New(senml.JSON) - msg := mainflux.Message{ + msg := messaging.Message{ Channel: "channel", Subtopic: "subtopic", Publisher: "publisher", @@ -55,7 +55,7 @@ func TestTransformJSON(t *testing.T) { cases := []struct { desc string - msg mainflux.Message + msg messaging.Message msgs interface{} err error }{ @@ -91,7 +91,7 @@ func TestTransformCBOR(t *testing.T) { require.Nil(t, err, "Decoding CBOR expected to succeed") tr := senml.New(senml.CBOR) - msg := mainflux.Message{ + msg := messaging.Message{ Channel: "channel", Subtopic: "subtopic", Publisher: "publisher", @@ -126,7 +126,7 @@ func TestTransformCBOR(t *testing.T) { cases := []struct { desc string - msg mainflux.Message + msg messaging.Message msgs interface{} err error }{ From f54047ffa9b4c41c4e6d41d7f8756377ce9d4a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 21:38:17 +0200 Subject: [PATCH 21/28] Fix message proto commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- Makefile | 2 +- message.pb.go | 574 ---------------------------------------- message.proto | 17 -- messaging/message.pb.go | 37 ++- pubsub.go | 26 -- 5 files changed, 19 insertions(+), 637 deletions(-) delete mode 100644 message.pb.go delete mode 100644 message.proto delete mode 100644 pubsub.go diff --git a/Makefile b/Makefile index fbaabd56c5..f5d7b6b7b6 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ test: proto: protoc --gofast_out=plugins=grpc:. *.proto - protoc --gofast_out=plugins=grpc:. messaging/*.proto + protoc --gogo_out=plugins=grpc:. messaging/*.proto $(SERVICES): $(call compile_service,$(@)) diff --git a/message.pb.go b/message.pb.go deleted file mode 100644 index 1e3194fdd6..0000000000 --- a/message.pb.go +++ /dev/null @@ -1,574 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: message.proto - -package mainflux - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// Message represents a message emitted by the Mainflux adapters layer. -type Message struct { - Channel string `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` - Subtopic string `protobuf:"bytes,2,opt,name=subtopic,proto3" json:"subtopic,omitempty"` - Publisher string `protobuf:"bytes,3,opt,name=publisher,proto3" json:"publisher,omitempty"` - Protocol string `protobuf:"bytes,4,opt,name=protocol,proto3" json:"protocol,omitempty"` - Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` - // Timestamp the Message occured in the system. - // By the default, occured represents Unix nanoseconds timestamp. - Occurred int64 `protobuf:"varint,6,opt,name=occurred,proto3" json:"occurred,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Message) Reset() { *m = Message{} } -func (m *Message) String() string { return proto.CompactTextString(m) } -func (*Message) ProtoMessage() {} -func (*Message) Descriptor() ([]byte, []int) { - return fileDescriptor_33c57e4bae7b9afd, []int{0} -} -func (m *Message) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Message.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Message) XXX_Merge(src proto.Message) { - xxx_messageInfo_Message.Merge(m, src) -} -func (m *Message) XXX_Size() int { - return m.Size() -} -func (m *Message) XXX_DiscardUnknown() { - xxx_messageInfo_Message.DiscardUnknown(m) -} - -var xxx_messageInfo_Message proto.InternalMessageInfo - -func (m *Message) GetChannel() string { - if m != nil { - return m.Channel - } - return "" -} - -func (m *Message) GetSubtopic() string { - if m != nil { - return m.Subtopic - } - return "" -} - -func (m *Message) GetPublisher() string { - if m != nil { - return m.Publisher - } - return "" -} - -func (m *Message) GetProtocol() string { - if m != nil { - return m.Protocol - } - return "" -} - -func (m *Message) GetPayload() []byte { - if m != nil { - return m.Payload - } - return nil -} - -func (m *Message) GetOccurred() int64 { - if m != nil { - return m.Occurred - } - return 0 -} - -func init() { - proto.RegisterType((*Message)(nil), "mainflux.Message") -} - -func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } - -var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 187 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, - 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xc8, 0x4d, 0xcc, 0xcc, 0x4b, - 0xcb, 0x29, 0xad, 0x50, 0x5a, 0xcf, 0xc8, 0xc5, 0xee, 0x0b, 0x91, 0x13, 0x92, 0xe0, 0x62, 0x4f, - 0xce, 0x48, 0xcc, 0xcb, 0x4b, 0xcd, 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0x71, 0x85, - 0xa4, 0xb8, 0x38, 0x8a, 0x4b, 0x93, 0x4a, 0xf2, 0x0b, 0x32, 0x93, 0x25, 0x98, 0xc0, 0x52, 0x70, - 0xbe, 0x90, 0x0c, 0x17, 0x67, 0x41, 0x69, 0x52, 0x4e, 0x66, 0x71, 0x46, 0x6a, 0x91, 0x04, 0x33, - 0x58, 0x12, 0x21, 0x00, 0xd2, 0x09, 0xb6, 0x32, 0x39, 0x3f, 0x47, 0x82, 0x05, 0xa2, 0x13, 0xc6, - 0x07, 0xd9, 0x57, 0x90, 0x58, 0x99, 0x93, 0x9f, 0x98, 0x22, 0xc1, 0xaa, 0xc0, 0xa8, 0xc1, 0x13, - 0x04, 0xe3, 0x82, 0x74, 0xe5, 0x27, 0x27, 0x97, 0x16, 0x15, 0xa5, 0xa6, 0x48, 0xb0, 0x29, 0x30, - 0x6a, 0x30, 0x07, 0xc1, 0xf9, 0x4e, 0x02, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7, 0xf8, - 0xe0, 0x91, 0x1c, 0xe3, 0x8c, 0xc7, 0x72, 0x0c, 0x49, 0x6c, 0x60, 0x13, 0x8d, 0x01, 0x01, 0x00, - 0x00, 0xff, 0xff, 0x81, 0x36, 0x5f, 0x25, 0xe5, 0x00, 0x00, 0x00, -} - -func (m *Message) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Message) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Message) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.Occurred != 0 { - i = encodeVarintMessage(dAtA, i, uint64(m.Occurred)) - i-- - dAtA[i] = 0x30 - } - if len(m.Payload) > 0 { - i -= len(m.Payload) - copy(dAtA[i:], m.Payload) - i = encodeVarintMessage(dAtA, i, uint64(len(m.Payload))) - i-- - dAtA[i] = 0x2a - } - if len(m.Protocol) > 0 { - i -= len(m.Protocol) - copy(dAtA[i:], m.Protocol) - i = encodeVarintMessage(dAtA, i, uint64(len(m.Protocol))) - i-- - dAtA[i] = 0x22 - } - if len(m.Publisher) > 0 { - i -= len(m.Publisher) - copy(dAtA[i:], m.Publisher) - i = encodeVarintMessage(dAtA, i, uint64(len(m.Publisher))) - i-- - dAtA[i] = 0x1a - } - if len(m.Subtopic) > 0 { - i -= len(m.Subtopic) - copy(dAtA[i:], m.Subtopic) - i = encodeVarintMessage(dAtA, i, uint64(len(m.Subtopic))) - i-- - dAtA[i] = 0x12 - } - if len(m.Channel) > 0 { - i -= len(m.Channel) - copy(dAtA[i:], m.Channel) - i = encodeVarintMessage(dAtA, i, uint64(len(m.Channel))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { - offset -= sovMessage(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *Message) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Channel) - if l > 0 { - n += 1 + l + sovMessage(uint64(l)) - } - l = len(m.Subtopic) - if l > 0 { - n += 1 + l + sovMessage(uint64(l)) - } - l = len(m.Publisher) - if l > 0 { - n += 1 + l + sovMessage(uint64(l)) - } - l = len(m.Protocol) - if l > 0 { - n += 1 + l + sovMessage(uint64(l)) - } - l = len(m.Payload) - if l > 0 { - n += 1 + l + sovMessage(uint64(l)) - } - if m.Occurred != 0 { - n += 1 + sovMessage(uint64(m.Occurred)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - -func sovMessage(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozMessage(x uint64) (n int) { - return sovMessage(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *Message) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Message: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Message: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Channel", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Channel = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Subtopic", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Subtopic = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Publisher", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Publisher = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Protocol", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Protocol = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Payload", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Payload = append(m.Payload[:0], dAtA[iNdEx:postIndex]...) - if m.Payload == nil { - m.Payload = []byte{} - } - iNdEx = postIndex - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Occurred", wireType) - } - m.Occurred = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Occurred |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipMessage(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthMessage - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthMessage - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipMessage(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMessage - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMessage - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMessage - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthMessage - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupMessage - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthMessage - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthMessage = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowMessage = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupMessage = fmt.Errorf("proto: unexpected end of group") -) diff --git a/message.proto b/message.proto deleted file mode 100644 index ce048434af..0000000000 --- a/message.proto +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Mainflux -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; -package mainflux; - -// Message represents a message emitted by the Mainflux adapters layer. -message Message { - string channel = 1; - string subtopic = 2; - string publisher = 3; - string protocol = 4; - bytes payload = 5; - // Timestamp the Message occured in the system. - // By the default, occured represents Unix nanoseconds timestamp. - int64 occurred = 6; -} diff --git a/messaging/message.pb.go b/messaging/message.pb.go index 4995187819..d6082fd2bc 100644 --- a/messaging/message.pb.go +++ b/messaging/message.pb.go @@ -5,10 +5,9 @@ package messaging import ( fmt "fmt" - math "math" - proto "github.com/gogo/protobuf/proto" timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -39,7 +38,7 @@ func (m *Message) Reset() { *m = Message{} } func (m *Message) String() string { return proto.CompactTextString(m) } func (*Message) ProtoMessage() {} func (*Message) Descriptor() ([]byte, []int) { - return fileDescriptor_6357da820a7eacc2, []int{0} + return fileDescriptor_77305e42dfa8cc5b, []int{0} } func (m *Message) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Message.Unmarshal(m, b) @@ -102,24 +101,24 @@ func (m *Message) GetCreated() *timestamp.Timestamp { } func init() { - proto.RegisterType((*Message)(nil), "broker.Message") + proto.RegisterType((*Message)(nil), "messaging.Message") } -func init() { proto.RegisterFile("broker/message.proto", fileDescriptor_6357da820a7eacc2) } +func init() { proto.RegisterFile("messaging/message.proto", fileDescriptor_77305e42dfa8cc5b) } -var fileDescriptor_6357da820a7eacc2 = []byte{ +var fileDescriptor_77305e42dfa8cc5b = []byte{ // 202 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x8f, 0xcb, 0x6e, 0x84, 0x20, - 0x18, 0x85, 0x43, 0x2f, 0x5a, 0x69, 0x57, 0xa4, 0x0b, 0x62, 0x9a, 0xd4, 0x74, 0xe5, 0x0a, 0x93, - 0xb6, 0xaf, 0x31, 0x1b, 0x33, 0x2f, 0x00, 0xf8, 0x8f, 0x9a, 0x41, 0x7f, 0x02, 0xb8, 0x98, 0x47, - 0x9c, 0xb7, 0x9a, 0x08, 0x61, 0x66, 0xf9, 0x9d, 0x4b, 0x4e, 0x0e, 0xfd, 0x54, 0x0e, 0xcf, 0xe0, - 0xba, 0x05, 0xbc, 0x97, 0x23, 0x08, 0xeb, 0x30, 0x20, 0x2b, 0x92, 0x5a, 0x7f, 0x8f, 0x88, 0xa3, - 0x81, 0x2e, 0xaa, 0x6a, 0x3b, 0x75, 0x61, 0x5e, 0xc0, 0x07, 0xb9, 0xd8, 0x14, 0xfc, 0xb9, 0x12, - 0x5a, 0x1e, 0x52, 0x95, 0x71, 0x5a, 0xea, 0x49, 0xae, 0x2b, 0x18, 0x4e, 0x1a, 0xd2, 0x56, 0x7d, - 0x46, 0x56, 0xd3, 0x37, 0xbf, 0xa9, 0x80, 0x76, 0xd6, 0xfc, 0x29, 0x5a, 0x77, 0x66, 0x5f, 0xb4, - 0xb2, 0x9b, 0x32, 0xb3, 0x9f, 0xc0, 0xf1, 0xe7, 0x68, 0x3e, 0x84, 0xbd, 0x19, 0x87, 0x34, 0x1a, - 0xfe, 0x92, 0x9a, 0x99, 0xf7, 0x3d, 0x2b, 0x2f, 0x06, 0xe5, 0xc0, 0x5f, 0x1b, 0xd2, 0x7e, 0xf4, - 0x19, 0xd9, 0x3f, 0x2d, 0xb5, 0x03, 0x19, 0x60, 0xe0, 0x45, 0x43, 0xda, 0xf7, 0xdf, 0x5a, 0xa4, - 0x23, 0x22, 0x1f, 0x11, 0xc7, 0x7c, 0xa4, 0xcf, 0x51, 0x55, 0x44, 0xf3, 0xef, 0x16, 0x00, 0x00, - 0xff, 0xff, 0x0b, 0xd8, 0x04, 0x71, 0x13, 0x01, 0x00, 0x00, + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x8f, 0xcd, 0x8e, 0x83, 0x20, + 0x14, 0x85, 0xc3, 0xfc, 0xe8, 0xc8, 0xcc, 0x8a, 0xcd, 0x10, 0x33, 0xc9, 0x98, 0xae, 0x5c, 0x61, + 0xd2, 0xf6, 0x35, 0xba, 0x31, 0x7d, 0x01, 0xc0, 0x5b, 0x24, 0x41, 0x21, 0x82, 0x8b, 0x3e, 0x62, + 0xdf, 0xaa, 0x11, 0x82, 0xdd, 0xf1, 0xf1, 0xdd, 0x93, 0x93, 0x83, 0x7f, 0x27, 0xf0, 0x9e, 0x2b, + 0x3d, 0xab, 0x2e, 0xbd, 0x80, 0xb9, 0xc5, 0x06, 0x4b, 0xaa, 0x5d, 0xd4, 0xff, 0xca, 0x5a, 0x65, + 0xa0, 0x8b, 0x42, 0xac, 0xb7, 0x2e, 0xe8, 0x09, 0x7c, 0xe0, 0x93, 0x4b, 0xb7, 0x87, 0x07, 0xc2, + 0xe5, 0x25, 0xa5, 0x09, 0xc5, 0xa5, 0x1c, 0xf9, 0x3c, 0x83, 0xa1, 0xa8, 0x41, 0x6d, 0xd5, 0x67, + 0x24, 0x35, 0xfe, 0xf2, 0xab, 0x08, 0xd6, 0x69, 0x49, 0xdf, 0xa2, 0xda, 0x99, 0xfc, 0xe1, 0xca, + 0xad, 0xc2, 0x68, 0x3f, 0xc2, 0x42, 0xdf, 0xa3, 0x7c, 0x7d, 0x6c, 0xc9, 0x58, 0x24, 0xad, 0xa1, + 0x1f, 0x29, 0x99, 0x79, 0xeb, 0x73, 0xfc, 0x6e, 0x2c, 0x1f, 0xe8, 0x67, 0x83, 0xda, 0x9f, 0x3e, + 0x23, 0x39, 0xe3, 0x52, 0x2e, 0xc0, 0x03, 0x0c, 0xb4, 0x68, 0x50, 0xfb, 0x7d, 0xac, 0x59, 0x1a, + 0xc2, 0xf2, 0x10, 0x76, 0xcd, 0x43, 0xfa, 0x7c, 0x2a, 0x8a, 0x28, 0x4f, 0xcf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x30, 0x7a, 0x2f, 0x67, 0x19, 0x01, 0x00, 0x00, } diff --git a/pubsub.go b/pubsub.go deleted file mode 100644 index 4f41b4adc2..0000000000 --- a/pubsub.go +++ /dev/null @@ -1,26 +0,0 @@ -package mainflux - -// Publisher specifies message publishing API. -type Publisher interface { - // Publishes message to the stream. - Publish(topic string, msg Message) error -} - -// MessageHandler represents Message handler for Subscriber. -type MessageHandler func(msg Message) error - -// Subscriber specifies message subscription API. -type Subscriber interface { - // Subscribe subscribes to the message stream and consumes messages. - Subscribe(topic string, handler MessageHandler) error - - // Unsubscribe unsubscribes from the message stream and - // stops consuming messages. - Unsubscribe(topic string) error -} - -// PubSub represents aggregation interface for publisher and subscriber. -type PubSub interface { - Publisher - Subscriber -} From 7f1e76ac74bfedabac3b8e83beb8c1216600beba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 21:41:51 +0200 Subject: [PATCH 22/28] Replace string literal with constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/twins/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/twins/main.go b/cmd/twins/main.go index fd53756f33..3933d6549d 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -262,7 +262,7 @@ func newService(ps messaging.PubSub, ncTracer opentracing.Tracer, chanID string, }, []string{"method"}), ) - err := ps.Subscribe("channels.>", func(msg messaging.Message) error { + err := ps.Subscribe(pubsub.SubjectAllChannels, func(msg messaging.Message) error { if msg.Channel == chanID { return nil } From 09165b779e32c3b3c268f207c1248ae147843d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 21:42:57 +0200 Subject: [PATCH 23/28] Remove alias from main method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/coap/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/coap/main.go b/cmd/coap/main.go index 9750192e14..852aff1346 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -23,7 +23,7 @@ import ( logger "github.com/mainflux/mainflux/logger" pubsub "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - broker "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -82,7 +82,7 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) respChan := make(chan string, 10000) - nc, err := broker.Connect(cfg.natsURL) + nc, err := nats.Connect(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) From 408c267870e09e88fe47f29d86389de43f8945b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Mon, 27 Apr 2020 21:55:07 +0200 Subject: [PATCH 24/28] Change messaging pubsub alias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 4 ++-- cmd/coap/main.go | 4 ++-- cmd/http/main.go | 4 ++-- cmd/influxdb-writer/main.go | 4 ++-- cmd/lora/main.go | 4 ++-- cmd/mongodb-writer/main.go | 4 ++-- cmd/opcua/main.go | 4 ++-- cmd/postgres-writer/main.go | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index ee014af033..35324d2615 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -17,7 +17,7 @@ import ( "github.com/gocql/gocql" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -76,7 +76,7 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) + ps := messaging.NewPubSub(nc, "", logger) session := connectToCassandra(cfg.dbCfg, logger) defer session.Close() diff --git a/cmd/coap/main.go b/cmd/coap/main.go index 852aff1346..94cc3d6410 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -21,7 +21,7 @@ import ( "github.com/mainflux/mainflux/coap" "github.com/mainflux/mainflux/coap/api" logger "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" @@ -89,7 +89,7 @@ func main() { } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) + ps := messaging.NewPubSub(nc, "", logger) svc := coap.New(ps, logger, cc, respChan) svc = api.LoggingMiddleware(svc, logger) diff --git a/cmd/http/main.go b/cmd/http/main.go index c300168e53..8889305168 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -22,7 +22,7 @@ import ( adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/http/api" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" @@ -85,7 +85,7 @@ func main() { os.Exit(1) } defer nc.Close() - pub := pubsub.NewPublisher(nc) + pub := messaging.NewPublisher(nc) tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) svc := adapter.New(pub, tc) diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index 95c7d743af..c2d13be1ff 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -15,7 +15,7 @@ import ( influxdata "github.com/influxdata/influxdb/client/v2" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -77,7 +77,7 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) + ps := messaging.NewPubSub(nc, "", logger) client, err := influxdata.NewHTTPClient(clientCfg) if err != nil { diff --git a/cmd/lora/main.go b/cmd/lora/main.go index 85539b0ea0..bebf56afce 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -19,7 +19,7 @@ import ( "github.com/mainflux/mainflux/lora" "github.com/mainflux/mainflux/lora/api" "github.com/mainflux/mainflux/lora/mqtt" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" @@ -92,7 +92,7 @@ func main() { os.Exit(1) } defer nc.Close() - pub := pubsub.NewPublisher(nc) + pub := messaging.NewPublisher(nc) thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) chanRM := newRouteMapRepositoy(rmConn, channelsRMPrefix, logger) diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 3e457b2e01..449fcfdfef 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -15,7 +15,7 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -73,7 +73,7 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) + ps := messaging.NewPubSub(nc, "", logger) addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index 13d1b199e1..86e957da88 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -16,7 +16,7 @@ import ( r "github.com/go-redis/redis" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/opcua" "github.com/mainflux/mainflux/opcua/api" "github.com/mainflux/mainflux/opcua/db" @@ -104,7 +104,7 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) + ps := messaging.NewPubSub(nc, "", logger) ctx := context.Background() sub := gopcua.NewSubscriber(ctx, ps, thingRM, chanRM, connRM, logger) diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index bf845ad670..213962a596 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -15,7 +15,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - pubsub "github.com/mainflux/mainflux/messaging/nats" + messaging "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" @@ -82,7 +82,7 @@ func main() { os.Exit(1) } defer nc.Close() - ps := pubsub.NewPubSub(nc, "", logger) + ps := messaging.NewPubSub(nc, "", logger) db := connectToDB(cfg.dbConfig, logger) defer db.Close() From d22e451e8c2c57448c85959595e191062f7ebe27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Tue, 28 Apr 2020 00:18:37 +0200 Subject: [PATCH 25/28] Rename occured to created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- coap/api/transport.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coap/api/transport.go b/coap/api/transport.go index 017487f691..e870db1914 100644 --- a/coap/api/transport.go +++ b/coap/api/transport.go @@ -225,7 +225,7 @@ func receive(svc coap.Service, msg *gocoap.Message) *gocoap.Message { return res } - occured, err := ptypes.TimestampProto(time.Now()) + created, err := ptypes.TimestampProto(time.Now()) if err != nil { return nil } @@ -236,7 +236,7 @@ func receive(svc coap.Service, msg *gocoap.Message) *gocoap.Message { Publisher: publisher, Protocol: protocol, Payload: msg.Payload, - Created: occured, + Created: created, } if err := svc.Publish(m); err != nil { From ae76b43bc7279d4181766a236c0cad9b89e62cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Tue, 28 Apr 2020 00:53:45 +0200 Subject: [PATCH 26/28] Handle NATS connection in the NATS PubSub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 10 +++---- cmd/coap/main.go | 10 +++---- cmd/http/main.go | 12 ++++---- cmd/influxdb-writer/main.go | 10 +++---- cmd/lora/main.go | 10 +++---- cmd/mongodb-writer/main.go | 10 +++---- cmd/mqtt/main.go | 10 +++---- cmd/opcua/main.go | 10 +++---- cmd/postgres-writer/main.go | 10 +++---- cmd/twins/main.go | 12 ++++---- messaging/nats/publisher.go | 20 +++++++++++-- messaging/nats/pubsub.go | 58 +++++++++++++++++++++++------------- 12 files changed, 97 insertions(+), 85 deletions(-) diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 35324d2615..4cdc53e1fb 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -17,12 +17,11 @@ import ( "github.com/gocql/gocql" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/cassandra" - "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -70,20 +69,19 @@ func main() { log.Fatalf(err.Error()) } - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - ps := messaging.NewPubSub(nc, "", logger) + defer n.Close() session := connectToCassandra(cfg.dbCfg, logger) defer session.Close() repo := newService(session, logger) st := senml.New(cfg.contentType) - if err := writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Cassandra writer: %s", err)) } diff --git a/cmd/coap/main.go b/cmd/coap/main.go index 94cc3d6410..9ce478e2b0 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -21,9 +21,8 @@ import ( "github.com/mainflux/mainflux/coap" "github.com/mainflux/mainflux/coap/api" logger "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -82,15 +81,14 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) respChan := make(chan string, 10000) - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() + defer n.Close() - ps := messaging.NewPubSub(nc, "", logger) - svc := coap.New(ps, logger, cc, respChan) + svc := coap.New(n, logger, cc, respChan) svc = api.LoggingMiddleware(svc, logger) diff --git a/cmd/http/main.go b/cmd/http/main.go index 8889305168..adfa2b775b 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -22,10 +22,9 @@ import ( adapter "github.com/mainflux/mainflux/http" "github.com/mainflux/mainflux/http/api" "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" - "github.com/nats-io/nats.go" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" @@ -79,16 +78,15 @@ func main() { thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) defer thingsCloser.Close() - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPublisher(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - pub := messaging.NewPublisher(nc) + defer n.Close() tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - svc := adapter.New(pub, tc) + svc := adapter.New(n, tc) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index c2d13be1ff..2c447f948a 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -15,12 +15,11 @@ import ( influxdata "github.com/influxdata/influxdb/client/v2" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/influxdb" - "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -71,13 +70,12 @@ func main() { log.Fatalf(err.Error()) } - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - ps := messaging.NewPubSub(nc, "", logger) + defer n.Close() client, err := influxdata.NewHTTPClient(clientCfg) if err != nil { @@ -93,7 +91,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start InfluxDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/lora/main.go b/cmd/lora/main.go index bebf56afce..56d80b26de 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -19,8 +19,7 @@ import ( "github.com/mainflux/mainflux/lora" "github.com/mainflux/mainflux/lora/api" "github.com/mainflux/mainflux/lora/mqtt" - messaging "github.com/mainflux/mainflux/messaging/nats" - "github.com/nats-io/nats.go" + "github.com/mainflux/mainflux/messaging/nats" kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux/lora/redis" @@ -86,20 +85,19 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPublisher(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - pub := messaging.NewPublisher(nc) + defer n.Close() thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) chanRM := newRouteMapRepositoy(rmConn, channelsRMPrefix, logger) mqttConn := connectToMQTTBroker(cfg.loraMsgURL, logger) - svc := lora.New(pub, thingRM, chanRM) + svc := lora.New(n, thingRM, chanRM) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( svc, diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 449fcfdfef..7327c34005 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -15,12 +15,11 @@ import ( kitprometheus "github.com/go-kit/kit/metrics/prometheus" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/mongodb" - "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -67,13 +66,12 @@ func main() { log.Fatal(err) } - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - ps := messaging.NewPubSub(nc, "", logger) + defer n.Close() addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) @@ -90,7 +88,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start MongoDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index 7c6bcf2403..08db68c111 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -16,14 +16,13 @@ import ( "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/messaging" - pubsub "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" mqtt "github.com/mainflux/mainflux/mqtt" mr "github.com/mainflux/mainflux/mqtt/redis" thingsapi "github.com/mainflux/mainflux/things/api/auth/grpc" mp "github.com/mainflux/mproxy/pkg/mqtt" "github.com/mainflux/mproxy/pkg/session" ws "github.com/mainflux/mproxy/pkg/websocket" - "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" jconfig "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" @@ -131,18 +130,17 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPublisher(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - pub := pubsub.NewPublisher(nc) + defer n.Close() es := mr.NewEventStore(rc, cfg.instance) // Event handler for MQTT hooks - evt := mqtt.New([]messaging.Publisher{pub}, cc, es, logger, tracer) + evt := mqtt.New([]messaging.Publisher{n}, cc, es, logger, tracer) errs := make(chan error, 2) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index 86e957da88..dce720cc89 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -16,13 +16,12 @@ import ( r "github.com/go-redis/redis" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/opcua" "github.com/mainflux/mainflux/opcua/api" "github.com/mainflux/mainflux/opcua/db" "github.com/mainflux/mainflux/opcua/gopcua" "github.com/mainflux/mainflux/opcua/redis" - "github.com/nats-io/nats.go" kitprometheus "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" @@ -98,16 +97,15 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - ps := messaging.NewPubSub(nc, "", logger) + defer n.Close() ctx := context.Background() - sub := gopcua.NewSubscriber(ctx, ps, thingRM, chanRM, connRM, logger) + sub := gopcua.NewSubscriber(ctx, n, thingRM, chanRM, connRM, logger) browser := gopcua.NewBrowser(ctx, logger) svc := opcua.New(sub, browser, thingRM, chanRM, connRM, cfg.opcuaConfig, logger) diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index 213962a596..ad0c542155 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -15,12 +15,11 @@ import ( "github.com/jmoiron/sqlx" "github.com/mainflux/mainflux" "github.com/mainflux/mainflux/logger" - messaging "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" "github.com/mainflux/mainflux/transformers/senml" "github.com/mainflux/mainflux/writers" "github.com/mainflux/mainflux/writers/api" "github.com/mainflux/mainflux/writers/postgres" - "github.com/nats-io/nats.go" stdprometheus "github.com/prometheus/client_golang/prometheus" ) @@ -76,20 +75,19 @@ func main() { log.Fatalf(err.Error()) } - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - ps := messaging.NewPubSub(nc, "", logger) + defer n.Close() db := connectToDB(cfg.dbConfig, logger) defer db.Close() repo := newService(db, logger) st := senml.New(cfg.contentType) - if err = writers.Start(ps, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err = writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) } diff --git a/cmd/twins/main.go b/cmd/twins/main.go index 3933d6549d..9ed0ecf0c6 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -20,14 +20,13 @@ import ( authapi "github.com/mainflux/mainflux/authn/api/grpc" "github.com/mainflux/mainflux/logger" "github.com/mainflux/mainflux/messaging" - pubsub "github.com/mainflux/mainflux/messaging/nats" + "github.com/mainflux/mainflux/messaging/nats" localusers "github.com/mainflux/mainflux/things/users" "github.com/mainflux/mainflux/twins" "github.com/mainflux/mainflux/twins/api" twapi "github.com/mainflux/mainflux/twins/api/http" twmongodb "github.com/mainflux/mainflux/twins/mongodb" "github.com/mainflux/mainflux/twins/uuid" - "github.com/nats-io/nats.go" opentracing "github.com/opentracing/opentracing-go" stdprometheus "github.com/prometheus/client_golang/prometheus" jconfig "github.com/uber/jaeger-client-go/config" @@ -115,13 +114,12 @@ func main() { dbTracer, dbCloser := initJaeger("twins_db", cfg.jaegerURL, logger) defer dbCloser.Close() - nc, err := nats.Connect(cfg.natsURL) + n, err := nats.NewPubSub(cfg.natsURL, queue, logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer nc.Close() - ps := pubsub.NewPubSub(nc, queue, logger) + defer n.Close() ncTracer, ncCloser := initJaeger("twins_nats", cfg.jaegerURL, logger) defer ncCloser.Close() @@ -129,7 +127,7 @@ func main() { tracer, closer := initJaeger("twins", cfg.jaegerURL, logger) defer closer.Close() - svc := newService(ps, ncTracer, cfg.channelID, auth, dbTracer, db, logger) + svc := newService(n, ncTracer, cfg.channelID, auth, dbTracer, db, logger) errs := make(chan error, 2) @@ -262,7 +260,7 @@ func newService(ps messaging.PubSub, ncTracer opentracing.Tracer, chanID string, }, []string{"method"}), ) - err := ps.Subscribe(pubsub.SubjectAllChannels, func(msg messaging.Message) error { + err := ps.Subscribe(nats.SubjectAllChannels, func(msg messaging.Message) error { if msg.Channel == chanID { return nil } diff --git a/messaging/nats/publisher.go b/messaging/nats/publisher.go index 56382b3afd..6472840234 100644 --- a/messaging/nats/publisher.go +++ b/messaging/nats/publisher.go @@ -17,11 +17,23 @@ type pubsliher struct { conn *broker.Conn } +// Publisher wraps messaging Publisher exposing +// Close() method for NATS connection. +type Publisher interface { + messaging.Publisher + Close() +} + // NewPublisher returns NATS message Publisher. -func NewPublisher(conn *broker.Conn) messaging.Publisher { - return &pubsub{ +func NewPublisher(url string) (Publisher, error) { + conn, err := broker.Connect(url) + if err != nil { + return nil, err + } + ret := &pubsub{ conn: conn, } + return ret, nil } func (pub *pubsliher) Publish(topic string, msg messaging.Message) error { @@ -40,3 +52,7 @@ func (pub *pubsliher) Publish(topic string, msg messaging.Message) error { return nil } + +func (pub *pubsliher) Close() { + pub.conn.Close() +} diff --git a/messaging/nats/pubsub.go b/messaging/nats/pubsub.go index 1b7e0e5b13..ff1fe10a81 100644 --- a/messaging/nats/pubsub.go +++ b/messaging/nats/pubsub.go @@ -28,6 +28,13 @@ var ( var _ messaging.PubSub = (*pubsub)(nil) +// PubSub wraps messaging Publisher exposing +// Close() method for NATS connection. +type PubSub interface { + messaging.PubSub + Close() +} + type pubsub struct { conn *broker.Conn logger log.Logger @@ -43,16 +50,21 @@ type pubsub struct { // from ordinary subscribe. For more information, please take a look // here: https://docs.nats.io/developing-with-nats/receiving/queues. // If the queue is empty, Subscribe will be used. -func NewPubSub(conn *broker.Conn, queue string, logger log.Logger) messaging.PubSub { - return &pubsub{ +func NewPubSub(url, queue string, logger log.Logger) (PubSub, error) { + conn, err := broker.Connect(url) + if err != nil { + return nil, err + } + ret := &pubsub{ conn: conn, queue: queue, logger: logger, subscriptions: make(map[string]*broker.Subscription), } + return ret, nil } -func (n *pubsub) Publish(topic string, msg messaging.Message) error { +func (ps *pubsub) Publish(topic string, msg messaging.Message) error { data, err := proto.Marshal(&msg) if err != nil { return err @@ -62,49 +74,49 @@ func (n *pubsub) Publish(topic string, msg messaging.Message) error { if msg.Subtopic != "" { subject = fmt.Sprintf("%s.%s", subject, msg.Subtopic) } - if err := n.conn.Publish(subject, data); err != nil { + if err := ps.conn.Publish(subject, data); err != nil { return err } return nil } -func (n *pubsub) Subscribe(topic string, handler messaging.MessageHandler) error { +func (ps *pubsub) Subscribe(topic string, handler messaging.MessageHandler) error { if topic == "" { return errEmptyTopic } - n.mu.Lock() - defer n.mu.Unlock() - if _, ok := n.subscriptions[topic]; ok { + ps.mu.Lock() + defer ps.mu.Unlock() + if _, ok := ps.subscriptions[topic]; ok { return errAlreadySubscribed } topic = fmt.Sprintf("%s.%s", chansPrefix, topic) - if n.queue != "" { - sub, err := n.conn.QueueSubscribe(topic, n.queue, n.natsHandler(handler)) + if ps.queue != "" { + sub, err := ps.conn.QueueSubscribe(topic, ps.queue, ps.natsHandler(handler)) if err != nil { return err } - n.subscriptions[topic] = sub + ps.subscriptions[topic] = sub return nil } - sub, err := n.conn.Subscribe(topic, n.natsHandler(handler)) + sub, err := ps.conn.Subscribe(topic, ps.natsHandler(handler)) if err != nil { return err } - n.subscriptions[topic] = sub + ps.subscriptions[topic] = sub return nil } -func (n *pubsub) Unsubscribe(topic string) error { +func (ps *pubsub) Unsubscribe(topic string) error { if topic == "" { return errEmptyTopic } - n.mu.Lock() - defer n.mu.Unlock() + ps.mu.Lock() + defer ps.mu.Unlock() topic = fmt.Sprintf("%s.%s", chansPrefix, topic) - sub, ok := n.subscriptions[topic] + sub, ok := ps.subscriptions[topic] if !ok { return errNotSubscribed } @@ -113,19 +125,23 @@ func (n *pubsub) Unsubscribe(topic string) error { return err } - delete(n.subscriptions, topic) + delete(ps.subscriptions, topic) return nil } -func (n *pubsub) natsHandler(h messaging.MessageHandler) broker.MsgHandler { +func (ps *pubsub) Close() { + ps.conn.Close() +} + +func (ps *pubsub) natsHandler(h messaging.MessageHandler) broker.MsgHandler { return func(m *broker.Msg) { var msg messaging.Message if err := proto.Unmarshal(m.Data, &msg); err != nil { - n.logger.Warn(fmt.Sprintf("Failed to unmarshal received message: %s", err)) + ps.logger.Warn(fmt.Sprintf("Failed to unmarshal received message: %s", err)) return } if err := h(msg); err != nil { - n.logger.Warn(fmt.Sprintf("Failed handle Mainflux message: %s", err)) + ps.logger.Warn(fmt.Sprintf("Failed handle Mainflux message: %s", err)) } } } From 0ca8c6043e6b86979c8659bb9a084357d846f8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Tue, 28 Apr 2020 01:51:05 +0200 Subject: [PATCH 27/28] Rename n to pub/pubSub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- cmd/cassandra-writer/main.go | 6 +++--- cmd/coap/main.go | 6 +++--- cmd/http/main.go | 6 +++--- cmd/influxdb-writer/main.go | 6 +++--- cmd/lora/main.go | 6 +++--- cmd/mongodb-writer/main.go | 6 +++--- cmd/mqtt/main.go | 6 +++--- cmd/opcua/main.go | 6 +++--- cmd/postgres-writer/main.go | 6 +++--- cmd/twins/main.go | 6 +++--- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/cmd/cassandra-writer/main.go b/cmd/cassandra-writer/main.go index 4cdc53e1fb..4158a09e00 100644 --- a/cmd/cassandra-writer/main.go +++ b/cmd/cassandra-writer/main.go @@ -69,19 +69,19 @@ func main() { log.Fatalf(err.Error()) } - n, err := nats.NewPubSub(cfg.natsURL, "", logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() session := connectToCassandra(cfg.dbCfg, logger) defer session.Close() repo := newService(session, logger) st := senml.New(cfg.contentType) - if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(pubSub, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Cassandra writer: %s", err)) } diff --git a/cmd/coap/main.go b/cmd/coap/main.go index 9ce478e2b0..2ab1bd24c6 100644 --- a/cmd/coap/main.go +++ b/cmd/coap/main.go @@ -81,14 +81,14 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) respChan := make(chan string, 10000) - n, err := nats.NewPubSub(cfg.natsURL, "", logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() - svc := coap.New(n, logger, cc, respChan) + svc := coap.New(pubSub, logger, cc, respChan) svc = api.LoggingMiddleware(svc, logger) diff --git a/cmd/http/main.go b/cmd/http/main.go index adfa2b775b..0c0f12a88f 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -78,15 +78,15 @@ func main() { thingsTracer, thingsCloser := initJaeger("things", cfg.jaegerURL, logger) defer thingsCloser.Close() - n, err := nats.NewPublisher(cfg.natsURL) + pub, err := nats.NewPublisher(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pub.Close() tc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - svc := adapter.New(n, tc) + svc := adapter.New(pub, tc) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( diff --git a/cmd/influxdb-writer/main.go b/cmd/influxdb-writer/main.go index 2c447f948a..d87029e511 100644 --- a/cmd/influxdb-writer/main.go +++ b/cmd/influxdb-writer/main.go @@ -70,12 +70,12 @@ func main() { log.Fatalf(err.Error()) } - n, err := nats.NewPubSub(cfg.natsURL, "", logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() client, err := influxdata.NewHTTPClient(clientCfg) if err != nil { @@ -91,7 +91,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(pubSub, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start InfluxDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/lora/main.go b/cmd/lora/main.go index 56d80b26de..3347812fb0 100644 --- a/cmd/lora/main.go +++ b/cmd/lora/main.go @@ -85,19 +85,19 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - n, err := nats.NewPublisher(cfg.natsURL) + pub, err := nats.NewPublisher(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pub.Close() thingRM := newRouteMapRepositoy(rmConn, thingsRMPrefix, logger) chanRM := newRouteMapRepositoy(rmConn, channelsRMPrefix, logger) mqttConn := connectToMQTTBroker(cfg.loraMsgURL, logger) - svc := lora.New(n, thingRM, chanRM) + svc := lora.New(pub, thingRM, chanRM) svc = api.LoggingMiddleware(svc, logger) svc = api.MetricsMiddleware( svc, diff --git a/cmd/mongodb-writer/main.go b/cmd/mongodb-writer/main.go index 7327c34005..0d463e1681 100644 --- a/cmd/mongodb-writer/main.go +++ b/cmd/mongodb-writer/main.go @@ -66,12 +66,12 @@ func main() { log.Fatal(err) } - n, err := nats.NewPubSub(cfg.natsURL, "", logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() addr := fmt.Sprintf("mongodb://%s:%s", cfg.dbHost, cfg.dbPort) client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(addr)) @@ -88,7 +88,7 @@ func main() { repo = api.MetricsMiddleware(repo, counter, latency) st := senml.New(cfg.contentType) - if err := writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err := writers.Start(pubSub, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to start MongoDB writer: %s", err)) os.Exit(1) } diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go index 08db68c111..aabaaab119 100644 --- a/cmd/mqtt/main.go +++ b/cmd/mqtt/main.go @@ -130,17 +130,17 @@ func main() { cc := thingsapi.NewClient(conn, thingsTracer, cfg.thingsAuthTimeout) - n, err := nats.NewPublisher(cfg.natsURL) + pub, err := nats.NewPublisher(cfg.natsURL) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pub.Close() es := mr.NewEventStore(rc, cfg.instance) // Event handler for MQTT hooks - evt := mqtt.New([]messaging.Publisher{n}, cc, es, logger, tracer) + evt := mqtt.New([]messaging.Publisher{pub}, cc, es, logger, tracer) errs := make(chan error, 2) diff --git a/cmd/opcua/main.go b/cmd/opcua/main.go index dce720cc89..40ebc002e8 100644 --- a/cmd/opcua/main.go +++ b/cmd/opcua/main.go @@ -97,15 +97,15 @@ func main() { esConn := connectToRedis(cfg.esURL, cfg.esPass, cfg.esDB, logger) defer esConn.Close() - n, err := nats.NewPubSub(cfg.natsURL, "", logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() ctx := context.Background() - sub := gopcua.NewSubscriber(ctx, n, thingRM, chanRM, connRM, logger) + sub := gopcua.NewSubscriber(ctx, pubSub, thingRM, chanRM, connRM, logger) browser := gopcua.NewBrowser(ctx, logger) svc := opcua.New(sub, browser, thingRM, chanRM, connRM, cfg.opcuaConfig, logger) diff --git a/cmd/postgres-writer/main.go b/cmd/postgres-writer/main.go index ad0c542155..708fc33d45 100644 --- a/cmd/postgres-writer/main.go +++ b/cmd/postgres-writer/main.go @@ -75,19 +75,19 @@ func main() { log.Fatalf(err.Error()) } - n, err := nats.NewPubSub(cfg.natsURL, "", logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, "", logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() db := connectToDB(cfg.dbConfig, logger) defer db.Close() repo := newService(db, logger) st := senml.New(cfg.contentType) - if err = writers.Start(n, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { + if err = writers.Start(pubSub, repo, st, svcName, cfg.subjectsCfgPath, logger); err != nil { logger.Error(fmt.Sprintf("Failed to create Postgres writer: %s", err)) } diff --git a/cmd/twins/main.go b/cmd/twins/main.go index 9ed0ecf0c6..83d4ab880e 100644 --- a/cmd/twins/main.go +++ b/cmd/twins/main.go @@ -114,12 +114,12 @@ func main() { dbTracer, dbCloser := initJaeger("twins_db", cfg.jaegerURL, logger) defer dbCloser.Close() - n, err := nats.NewPubSub(cfg.natsURL, queue, logger) + pubSub, err := nats.NewPubSub(cfg.natsURL, queue, logger) if err != nil { logger.Error(fmt.Sprintf("Failed to connect to NATS: %s", err)) os.Exit(1) } - defer n.Close() + defer pubSub.Close() ncTracer, ncCloser := initJaeger("twins_nats", cfg.jaegerURL, logger) defer ncCloser.Close() @@ -127,7 +127,7 @@ func main() { tracer, closer := initJaeger("twins", cfg.jaegerURL, logger) defer closer.Close() - svc := newService(n, ncTracer, cfg.channelID, auth, dbTracer, db, logger) + svc := newService(pubSub, ncTracer, cfg.channelID, auth, dbTracer, db, logger) errs := make(chan error, 2) From c73fb47cdc859c7226fda3e18e59f9b6ddb4108e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Borov=C4=8Danin?= Date: Tue, 28 Apr 2020 10:52:09 +0200 Subject: [PATCH 28/28] Fix typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dušan Borovčanin --- messaging/nats/publisher.go | 8 ++++---- messaging/nats/pubsub.go | 11 ++++++----- mqtt/adapter.go | 8 ++++---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/messaging/nats/publisher.go b/messaging/nats/publisher.go index 6472840234..cb81dd446a 100644 --- a/messaging/nats/publisher.go +++ b/messaging/nats/publisher.go @@ -11,9 +11,9 @@ import ( broker "github.com/nats-io/nats.go" ) -var _ messaging.Publisher = (*pubsliher)(nil) +var _ messaging.Publisher = (*publisher)(nil) -type pubsliher struct { +type publisher struct { conn *broker.Conn } @@ -36,7 +36,7 @@ func NewPublisher(url string) (Publisher, error) { return ret, nil } -func (pub *pubsliher) Publish(topic string, msg messaging.Message) error { +func (pub *publisher) Publish(topic string, msg messaging.Message) error { data, err := proto.Marshal(&msg) if err != nil { return err @@ -53,6 +53,6 @@ func (pub *pubsliher) Publish(topic string, msg messaging.Message) error { return nil } -func (pub *pubsliher) Close() { +func (pub *publisher) Close() { pub.conn.Close() } diff --git a/messaging/nats/pubsub.go b/messaging/nats/pubsub.go index ff1fe10a81..d6b025e0d8 100644 --- a/messaging/nats/pubsub.go +++ b/messaging/nats/pubsub.go @@ -44,9 +44,9 @@ type pubsub struct { } // NewPubSub returns NATS message publisher/subscriber. -// Paramter queue specifies the queue for the Subscribe method. +// Parameter queue specifies the queue for the Subscribe method. // If queue is specified (is not an empty string), Subscribe method -// will execute NATS QueueSubscibe which is conceptually different +// will execute NATS QueueSubscribe which is conceptually different // from ordinary subscribe. For more information, please take a look // here: https://docs.nats.io/developing-with-nats/receiving/queues. // If the queue is empty, Subscribe will be used. @@ -90,16 +90,17 @@ func (ps *pubsub) Subscribe(topic string, handler messaging.MessageHandler) erro if _, ok := ps.subscriptions[topic]; ok { return errAlreadySubscribed } + nh := ps.natsHandler(handler) topic = fmt.Sprintf("%s.%s", chansPrefix, topic) if ps.queue != "" { - sub, err := ps.conn.QueueSubscribe(topic, ps.queue, ps.natsHandler(handler)) + sub, err := ps.conn.QueueSubscribe(topic, ps.queue, nh) if err != nil { return err } ps.subscriptions[topic] = sub return nil } - sub, err := ps.conn.Subscribe(topic, ps.natsHandler(handler)) + sub, err := ps.conn.Subscribe(topic, nh) if err != nil { return err } @@ -141,7 +142,7 @@ func (ps *pubsub) natsHandler(h messaging.MessageHandler) broker.MsgHandler { return } if err := h(msg); err != nil { - ps.logger.Warn(fmt.Sprintf("Failed handle Mainflux message: %s", err)) + ps.logger.Warn(fmt.Sprintf("Failed to handle Mainflux message: %s", err)) } } } diff --git a/mqtt/adapter.go b/mqtt/adapter.go index 60e6e36329..003b2343ff 100644 --- a/mqtt/adapter.go +++ b/mqtt/adapter.go @@ -117,7 +117,7 @@ func (h *handler) AuthSubscribe(c *session.Client, topics *[]string) error { return nil } -// Connect - after client sucesfully connected +// Connect - after client successfully connected func (h *handler) Connect(c *session.Client) { if c == nil { h.logger.Error("Nil client connect") @@ -126,7 +126,7 @@ func (h *handler) Connect(c *session.Client) { h.logger.Info("Connect - client with ID: " + c.ID) } -// Publish - after client sucesfully published +// Publish - after client successfully published func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { if c == nil { h.logger.Error("Nil client publish") @@ -173,7 +173,7 @@ func (h *handler) Publish(c *session.Client, topic *string, payload *[]byte) { } } -// Subscribe - after client sucesfully subscribed +// Subscribe - after client successfully subscribed func (h *handler) Subscribe(c *session.Client, topics *[]string) { if c == nil { h.logger.Error("Nil client subscribe") @@ -188,7 +188,7 @@ func (h *handler) Unsubscribe(c *session.Client, topics *[]string) { h.logger.Error("Nil client unsubscribe") return } - h.logger.Info("Unubscribe - client ID: " + c.ID + ", form topics: " + strings.Join(*topics, ",")) + h.logger.Info("Unsubscribe - client ID: " + c.ID + ", form topics: " + strings.Join(*topics, ",")) } // Disconnect - connection with broker or client lost