From 76e2cd242ca30ebf40dc26a04a131bbb0a2afe6c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 12 Jan 2021 11:36:43 +0100 Subject: [PATCH 01/17] wip --- examples/metrics/xcloud.toml | 10 +++ pkg/metrics/config/config.go | 3 + pkg/metrics/driver/loader/loader.go | 1 + pkg/metrics/driver/xcloud/xcloud.go | 134 ++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+) create mode 100644 examples/metrics/xcloud.toml create mode 100644 pkg/metrics/driver/xcloud/xcloud.go diff --git a/examples/metrics/xcloud.toml b/examples/metrics/xcloud.toml new file mode 100644 index 0000000000..93d0f3f122 --- /dev/null +++ b/examples/metrics/xcloud.toml @@ -0,0 +1,10 @@ +[http] +address = "0.0.0.0:5550" + +[http.services.metrics] +metrics_data_driver_type = "xcloud" +metrics_record_interval = 5000 +cloud_instance="http://localhost/nextcloud" + +[http.services.prometheus] + diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index f4d6dbe3b2..0a213956a8 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -23,6 +23,8 @@ type Config struct { MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` MetricsDataLocation string `mapstructure:"metrics_data_location"` MetricsRecordInterval int `mapstructure:"metrics_record_interval"` + CloudInstance string `mapstructure:"cloud_instance"` + RegisterService string `mapstructure:"register_service"` } // Init sets sane defaults @@ -33,6 +35,7 @@ func (c *Config) Init() { c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" } } + if c.MetricsRecordInterval == 0 { c.MetricsRecordInterval = 5000 } diff --git a/pkg/metrics/driver/loader/loader.go b/pkg/metrics/driver/loader/loader.go index 97adaf9278..fcc6a1f9db 100644 --- a/pkg/metrics/driver/loader/loader.go +++ b/pkg/metrics/driver/loader/loader.go @@ -22,5 +22,6 @@ import ( // Load metrics drivers. _ "github.com/cs3org/reva/pkg/metrics/driver/dummy" _ "github.com/cs3org/reva/pkg/metrics/driver/json" + _ "github.com/cs3org/reva/pkg/metrics/driver/xcloud" // Add your own here ) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go new file mode 100644 index 0000000000..72e71781f2 --- /dev/null +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -0,0 +1,134 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package json + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + + "github.com/cs3org/reva/pkg/metrics/driver/registry" + + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/rs/zerolog" +) + +var log zerolog.Logger + +func init() { + log = logger.New().With().Int("pid", os.Getpid()).Logger() + driver := &CloudDriver{CloudData: &CloudData{}} + registry.Register(driverName(), driver) + + // TODO(labkode): spawn goroutines to fetch metrics and update the register service + + // get configuration from internal_metrics endpoint exposed + // by the sciencemesh app + + client := &http.Client{} + + // endpoint example: https://mybox.com or https://mybox.com/owncloud + endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", driver.CloudInstance) + + req, err := http.NewRequest("GET", endpoint, nil) + resp, err := client.Do(req) + if err != nil { + log.Err(err).Msgf("xcloud: error getting internal metrics from %s", driver.CloudInstance) + return + } + + if resp.StatusCode != http.StatusOK { + err := errors.New(fmt.Sprintf("xcloud: error getting internal metrics from %s", driver.CloudInstance)) + log.Err(err).Msgf("xcloud: error getting internal metrics from %s", driver.CloudInstance) + return + } + defer resp.Body.Close() + + // read response body + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", driver.CloudInstance) + } + + cd := &CloudData{} + if err := json.Unmarshal(data, cd); err != nil { + log.Err(err).Msgf("xcloud: error parsing body from internal metrics: body(%s)", string(data)) + } + + fmt.Printf("%+v\n", cd) +} + +func driverName() string { + return "xcloud" +} + +// CloudDriver the JsonDriver struct +type CloudDriver struct { + CloudInstance string + RegisterService string + CloudData *CloudData +} + +// Configure configures this driver +func (d *CloudDriver) Configure(c *config.Config) error { + if c.CloudInstance == "" { + err := errors.New("xcloud: missing cloud_instance config parameter") + return err + } + + d.CloudInstance = c.CloudInstance + return nil +} + +// GetNumUsers returns the number of site users +func (d *CloudDriver) GetNumUsers() int64 { + return d.CloudData.Metrics.TotalUsers +} + +// GetNumGroups returns the number of site groups +func (d *CloudDriver) GetNumGroups() int64 { + return d.CloudData.Metrics.TotalGroups +} + +// GetAmountStorage returns the amount of site storage used +func (d *CloudDriver) GetAmountStorage() int64 { + return d.CloudData.Metrics.TotalStorage +} + +type CloudData struct { + Metrics CloudDataMetrics `json:"metrics"` + Settings CloudDataSettings `json:"settings"` +} + +type CloudDataMetrics struct { + TotalUsers int64 `json:"total_users"` + TotalGroups int64 `json:"total_groups"` + TotalStorage int64 `json:"total_storage"` +} + +type CloudDataSettings struct { + IOPUrl string `json:"iopurl"` + Sitename string `json:"sitename"` + Hostname string `json:"hostname"` + Country string `json:"country"` +} From 6d92ab7161be68d651f7aaf453bdeefd679747bc Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Wed, 13 Jan 2021 09:24:42 +0000 Subject: [PATCH 02/17] fix lint --- pkg/metrics/driver/xcloud/xcloud.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 72e71781f2..1f70fed9eb 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -51,6 +51,11 @@ func init() { endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", driver.CloudInstance) req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + log.Err(err).Msgf("xcloud: error creating request to %s", driver.CloudInstance) + return + } + resp, err := client.Do(req) if err != nil { log.Err(err).Msgf("xcloud: error getting internal metrics from %s", driver.CloudInstance) @@ -58,7 +63,7 @@ func init() { } if resp.StatusCode != http.StatusOK { - err := errors.New(fmt.Sprintf("xcloud: error getting internal metrics from %s", driver.CloudInstance)) + err := fmt.Errorf("xcloud: error getting internal metrics from %s") log.Err(err).Msgf("xcloud: error getting internal metrics from %s", driver.CloudInstance) return } From 5126fe72cc5f517d1581a2cb15f5c0b963f61be0 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Thu, 14 Jan 2021 13:42:34 +0000 Subject: [PATCH 03/17] read internal metrics --- examples/metrics/xcloud.toml | 3 +- pkg/metrics/config/config.go | 5 +- pkg/metrics/driver/xcloud/xcloud.go | 82 ++++++++++++++++++++--------- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/examples/metrics/xcloud.toml b/examples/metrics/xcloud.toml index 93d0f3f122..7699aa9ad5 100644 --- a/examples/metrics/xcloud.toml +++ b/examples/metrics/xcloud.toml @@ -4,7 +4,8 @@ address = "0.0.0.0:5550" [http.services.metrics] metrics_data_driver_type = "xcloud" metrics_record_interval = 5000 -cloud_instance="http://localhost/nextcloud" +xcloud_instance="http://localhost" +xcloud_interval=5 [http.services.prometheus] diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index 0a213956a8..f21276d01c 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -23,8 +23,9 @@ type Config struct { MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` MetricsDataLocation string `mapstructure:"metrics_data_location"` MetricsRecordInterval int `mapstructure:"metrics_record_interval"` - CloudInstance string `mapstructure:"cloud_instance"` - RegisterService string `mapstructure:"register_service"` + XcloudInstance string `mapstructure:"xcloud_instance"` + XcloudCatalog string `mapstructure:"xcloud_catalog"` + XcloudPullInterval int `mapstructure:"xcloud_pull_interval"` } // Init sets sane defaults diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 1f70fed9eb..f78ea41010 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -21,8 +21,10 @@ package json import ( "encoding/json" "errors" + "sync" "fmt" "io/ioutil" + "time" "net/http" "os" @@ -40,68 +42,98 @@ func init() { driver := &CloudDriver{CloudData: &CloudData{}} registry.Register(driverName(), driver) +} + +func driverName() string { + return "xcloud" +} + +// CloudDriver the JsonDriver struct +type CloudDriver struct { + instance string + catalog string + pullInterval int + CloudData *CloudData + sync.Mutex +} + +func (d *CloudDriver) refresh() error { // TODO(labkode): spawn goroutines to fetch metrics and update the register service // get configuration from internal_metrics endpoint exposed // by the sciencemesh app - client := &http.Client{} // endpoint example: https://mybox.com or https://mybox.com/owncloud - endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", driver.CloudInstance) + endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics",d.instance) req, err := http.NewRequest("GET", endpoint, nil) if err != nil { - log.Err(err).Msgf("xcloud: error creating request to %s", driver.CloudInstance) - return + log.Err(err).Msgf("xcloud: error creating request to %s",d.instance) + return err } resp, err := client.Do(req) if err != nil { - log.Err(err).Msgf("xcloud: error getting internal metrics from %s", driver.CloudInstance) - return + log.Err(err).Msgf("xcloud: error getting internal metrics from %s",d.instance) + return err } if resp.StatusCode != http.StatusOK { - err := fmt.Errorf("xcloud: error getting internal metrics from %s") - log.Err(err).Msgf("xcloud: error getting internal metrics from %s", driver.CloudInstance) - return + err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", resp.StatusCode) + log.Err(err).Msgf("xcloud: error getting internal metrics from %s",d.instance) + return err } defer resp.Body.Close() // read response body data, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", driver.CloudInstance) + log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", d.instance) + return err } cd := &CloudData{} if err := json.Unmarshal(data, cd); err != nil { log.Err(err).Msgf("xcloud: error parsing body from internal metrics: body(%s)", string(data)) + return err } - fmt.Printf("%+v\n", cd) -} + d.Lock() + defer d.Unlock() + d.CloudData = cd + log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd) + return nil -func driverName() string { - return "xcloud" } - -// CloudDriver the JsonDriver struct -type CloudDriver struct { - CloudInstance string - RegisterService string - CloudData *CloudData -} - // Configure configures this driver func (d *CloudDriver) Configure(c *config.Config) error { - if c.CloudInstance == "" { - err := errors.New("xcloud: missing cloud_instance config parameter") + if c.XcloudInstance == "" { + err := errors.New("xcloud: missing xcloud_instance config parameter") return err } - d.CloudInstance = c.CloudInstance + if c.XcloudPullInterval == 0 { + c.XcloudPullInterval = 10 // seconds + } + + d.instance = c.XcloudInstance + d.pullInterval = c.XcloudPullInterval + + ticker := time.NewTicker(5 * time.Second) + quit := make(chan struct{}) + go func() { + for { + select { + case <- ticker.C: + d.refresh() + case <- quit: + ticker.Stop() + return + } + } + }() + return nil } From d931e017b874516970dbd1c2751ce540a4b0a554 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 09:37:57 +0000 Subject: [PATCH 04/17] wip --- pkg/metrics/driver/xcloud/xcloud.go | 122 +++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 21 deletions(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index f78ea41010..5eacd90065 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -21,12 +21,12 @@ package json import ( "encoding/json" "errors" - "sync" "fmt" "io/ioutil" - "time" "net/http" "os" + "sync" + "time" "github.com/cs3org/reva/pkg/metrics/driver/registry" @@ -50,10 +50,10 @@ func driverName() string { // CloudDriver the JsonDriver struct type CloudDriver struct { - instance string - catalog string - pullInterval int - CloudData *CloudData + instance string + catalog string + pullInterval int + CloudData *CloudData sync.Mutex } @@ -65,23 +65,23 @@ func (d *CloudDriver) refresh() error { client := &http.Client{} // endpoint example: https://mybox.com or https://mybox.com/owncloud - endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics",d.instance) + endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance) req, err := http.NewRequest("GET", endpoint, nil) if err != nil { - log.Err(err).Msgf("xcloud: error creating request to %s",d.instance) + log.Err(err).Msgf("xcloud: error creating request to %s", d.instance) return err } resp, err := client.Do(req) if err != nil { - log.Err(err).Msgf("xcloud: error getting internal metrics from %s",d.instance) + log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance) return err } if resp.StatusCode != http.StatusOK { err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", resp.StatusCode) - log.Err(err).Msgf("xcloud: error getting internal metrics from %s",d.instance) + log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance) return err } defer resp.Body.Close() @@ -103,9 +103,64 @@ func (d *CloudDriver) refresh() error { defer d.Unlock() d.CloudData = cd log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd) + + // if catalog is set, register site continously, this helps to also monitorize the health + // "xcloud: received internal metrics from cloud provider: &{Metrics:{TotalUsers:1 TotalGroups:0 TotalStorage:0} Settings:{IOPUrl:http://localhost:5550 Sitename:CERN Hostname:localhost Country:CH}}" + // + /* + { + "Name": "OC-Test@WWU", + "FullName": "ownCloud Test at University of Muenster", + "Homepage": "http://oc-test.uni-muenster.de", + "Description": "ownCloud Test Instance of University of Muenster", + "CountryCode": "DE", + "Services": [ + { + "Type": { + "Name": "REVAD" + }, + "Name": "oc-test.uni-muenster.de - REVAD", + "URL": "https://oc-test.uni-muenster.de/revad", + "IsMonitored": true, + "Properties": { + "METRICS_PATH": "/revad/metrics" + }, + "Host": "octest-test.uni-muenster.de" + } + ] + } + */ + + mc := &MentixCatalog{ + Name: cd.Settings.Sitename, + FullName: cd.Settings.Sitename, + Homepage: cd.Settings.Hostname, + Description: "ScienceMesh App from " + cd.Settings.Sitename, + CountryCode: cd.Settings.Country, + Services: []*MentixService{ + &MentixService{ + Host: cd.Settings.Hostname, + IsMonitored: true, + Name: cd.Settings.Hostname + " - REVAD", + URL: " + + /* + & + Host string `json:"Host"` + IsMonitored bool `json:"IsMonitored"` + Name string `json:"Name"` + Properties *MentixServiceProperties `json:"Properties"` + Type *MentixServiceType `json:"Type"` + URL string `json:"URL"` + */ + }, + }, + } + return nil } + // Configure configures this driver func (d *CloudDriver) Configure(c *config.Config) error { if c.XcloudInstance == "" { @@ -113,26 +168,26 @@ func (d *CloudDriver) Configure(c *config.Config) error { return err } - if c.XcloudPullInterval == 0 { + if c.XcloudPullInterval == 0 { c.XcloudPullInterval = 10 // seconds } d.instance = c.XcloudInstance d.pullInterval = c.XcloudPullInterval - ticker := time.NewTicker(5 * time.Second) + ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second) quit := make(chan struct{}) go func() { - for { - select { - case <- ticker.C: - d.refresh() - case <- quit: - ticker.Stop() - return + for { + select { + case <-ticker.C: + d.refresh() + case <-quit: + ticker.Stop() + return + } } - } - }() + }() return nil } @@ -169,3 +224,28 @@ type CloudDataSettings struct { Hostname string `json:"hostname"` Country string `json:"country"` } + +type MentixCatalog struct { + CountryCode string `json:"CountryCode"` + Description string `json:"Description"` + FullName string `json:"FullName"` + Homepage string `json:"Homepage"` + Name string `json:"Name"` + Services []*MentixService `json:"Services"` +} + +type MentixService struct { + Host string `json:"Host"` + IsMonitored bool `json:"IsMonitored"` + Name string `json:"Name"` + Properties *MentixServiceProperties `json:"Properties"` + Type *MentixServiceType `json:"Type"` + URL string `json:"URL"` +} + +type MentixServiceProperties struct { + MetricsPath string `json:"METRICS_PATH"` +} +type MentixServiceType struct { + Name string `json:"Name"` +} From 7bbe10053620dd3df4b511a203da1f543877a92b Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 11:28:26 +0000 Subject: [PATCH 05/17] basic reporting --- examples/metrics/xcloud.toml | 1 + pkg/metrics/config/config.go | 6 +-- pkg/metrics/driver/xcloud/xcloud.go | 74 +++++++++++++---------------- 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/examples/metrics/xcloud.toml b/examples/metrics/xcloud.toml index 7699aa9ad5..432e8d2c5b 100644 --- a/examples/metrics/xcloud.toml +++ b/examples/metrics/xcloud.toml @@ -6,6 +6,7 @@ metrics_data_driver_type = "xcloud" metrics_record_interval = 5000 xcloud_instance="http://localhost" xcloud_interval=5 +xcloud_catalog='https://sciencemesh-test.uni-muenster.de/api/mentix/?action=register' [http.services.prometheus] diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index f21276d01c..98ebe0070b 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -23,9 +23,9 @@ type Config struct { MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` MetricsDataLocation string `mapstructure:"metrics_data_location"` MetricsRecordInterval int `mapstructure:"metrics_record_interval"` - XcloudInstance string `mapstructure:"xcloud_instance"` - XcloudCatalog string `mapstructure:"xcloud_catalog"` - XcloudPullInterval int `mapstructure:"xcloud_pull_interval"` + XcloudInstance string `mapstructure:"xcloud_instance"` + XcloudCatalog string `mapstructure:"xcloud_catalog"` + XcloudPullInterval int `mapstructure:"xcloud_pull_interval"` } // Init sets sane defaults diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 5eacd90065..e543fd86fe 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -19,6 +19,7 @@ package json import ( + "bytes" "encoding/json" "errors" "fmt" @@ -104,33 +105,6 @@ func (d *CloudDriver) refresh() error { d.CloudData = cd log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd) - // if catalog is set, register site continously, this helps to also monitorize the health - // "xcloud: received internal metrics from cloud provider: &{Metrics:{TotalUsers:1 TotalGroups:0 TotalStorage:0} Settings:{IOPUrl:http://localhost:5550 Sitename:CERN Hostname:localhost Country:CH}}" - // - /* - { - "Name": "OC-Test@WWU", - "FullName": "ownCloud Test at University of Muenster", - "Homepage": "http://oc-test.uni-muenster.de", - "Description": "ownCloud Test Instance of University of Muenster", - "CountryCode": "DE", - "Services": [ - { - "Type": { - "Name": "REVAD" - }, - "Name": "oc-test.uni-muenster.de - REVAD", - "URL": "https://oc-test.uni-muenster.de/revad", - "IsMonitored": true, - "Properties": { - "METRICS_PATH": "/revad/metrics" - }, - "Host": "octest-test.uni-muenster.de" - } - ] - } - */ - mc := &MentixCatalog{ Name: cd.Settings.Sitename, FullName: cd.Settings.Sitename, @@ -139,26 +113,42 @@ func (d *CloudDriver) refresh() error { CountryCode: cd.Settings.Country, Services: []*MentixService{ &MentixService{ - Host: cd.Settings.Hostname, + Host: cd.Settings.Hostname, IsMonitored: true, - Name: cd.Settings.Hostname + " - REVAD", - URL: " - - /* - & - Host string `json:"Host"` - IsMonitored bool `json:"IsMonitored"` - Name string `json:"Name"` - Properties *MentixServiceProperties `json:"Properties"` - Type *MentixServiceType `json:"Type"` - URL string `json:"URL"` - */ + Name: cd.Settings.Hostname + " - REVAD", + URL: cd.Settings.Siteurl, + Properties: &MentixServiceProperties{ + MetricsPath: fmt.Sprintf("%s/index.php/apps/sciencemesh/metrics", cd.Settings.Siteurl), + }, + Type: &MentixServiceType{ + Name: "REVAD", + }, }, }, } - return nil + j, err := json.Marshal(mc) + if err != nil { + log.Err(err).Msgf("xcloud: error marhsaling mentix calalog info") + return err + } + log.Info().Msgf("xcloud: info to send to register: %s", string(j)) + + // send to register if catalog is set + req, err = http.NewRequest("POST", d.catalog, bytes.NewBuffer(j)) + resp, err = client.Do(req) + if err != nil { + log.Err(err).Msgf("xcloud: error registering catalog info to: %s with info: %s", d.catalog, string(j)) + return err + } + + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + + log.Info().Msgf("xcloud: site registered: %s", string(body)) + + return nil } // Configure configures this driver @@ -174,6 +164,7 @@ func (d *CloudDriver) Configure(c *config.Config) error { d.instance = c.XcloudInstance d.pullInterval = c.XcloudPullInterval + d.catalog = c.XcloudCatalog ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second) quit := make(chan struct{}) @@ -221,6 +212,7 @@ type CloudDataMetrics struct { type CloudDataSettings struct { IOPUrl string `json:"iopurl"` Sitename string `json:"sitename"` + Siteurl string `json:"siteurl"` Hostname string `json:"hostname"` Country string `json:"country"` } From 6e4d4410332277264291cf9f721433c4ccf10f3c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 13:48:21 +0000 Subject: [PATCH 06/17] skip insecurity verify --- pkg/metrics/driver/xcloud/xcloud.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index e543fd86fe..9ac615005c 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -22,6 +22,7 @@ import ( "bytes" "encoding/json" "errors" + "crypto/tls" "fmt" "io/ioutil" "net/http" @@ -63,7 +64,10 @@ func (d *CloudDriver) refresh() error { // get configuration from internal_metrics endpoint exposed // by the sciencemesh app - client := &http.Client{} + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} // endpoint example: https://mybox.com or https://mybox.com/owncloud endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance) @@ -118,7 +122,7 @@ func (d *CloudDriver) refresh() error { Name: cd.Settings.Hostname + " - REVAD", URL: cd.Settings.Siteurl, Properties: &MentixServiceProperties{ - MetricsPath: fmt.Sprintf("%s/index.php/apps/sciencemesh/metrics", cd.Settings.Siteurl), + MetricsPath: "/index.php/apps/sciencemesh/metrics", }, Type: &MentixServiceType{ Name: "REVAD", From 7f5b7dd5aa766e2b7ac0fcfef6c307fc663e492b Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 14:12:42 +0000 Subject: [PATCH 07/17] fix lint --- pkg/metrics/driver/xcloud/xcloud.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 9ac615005c..c1a9cfaa80 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -50,7 +50,7 @@ func driverName() string { return "xcloud" } -// CloudDriver the JsonDriver struct +// CloudDriver is the driver to use for Sciencemesh apps type CloudDriver struct { instance string catalog string @@ -202,17 +202,20 @@ func (d *CloudDriver) GetAmountStorage() int64 { return d.CloudData.Metrics.TotalStorage } +// CloudData represents the information obtained from the sciencemesh app type CloudData struct { Metrics CloudDataMetrics `json:"metrics"` Settings CloudDataSettings `json:"settings"` } +// CloudDataMetrics reprents the metrics gathered from the sciencemesh app type CloudDataMetrics struct { TotalUsers int64 `json:"total_users"` TotalGroups int64 `json:"total_groups"` TotalStorage int64 `json:"total_storage"` } +// CloudDataSettings represents the metrics gathered type CloudDataSettings struct { IOPUrl string `json:"iopurl"` Sitename string `json:"sitename"` @@ -221,6 +224,7 @@ type CloudDataSettings struct { Country string `json:"country"` } +// MentixCatalog represents the information needed to register a site into the mesh type MentixCatalog struct { CountryCode string `json:"CountryCode"` Description string `json:"Description"` @@ -230,6 +234,7 @@ type MentixCatalog struct { Services []*MentixService `json:"Services"` } +// MentixService represents the service running in a site type MentixService struct { Host string `json:"Host"` IsMonitored bool `json:"IsMonitored"` @@ -239,9 +244,12 @@ type MentixService struct { URL string `json:"URL"` } +// MentixSiteProperties represents the properties to expose the metrics endpoint type MentixServiceProperties struct { MetricsPath string `json:"METRICS_PATH"` } + +// MentixServiceType represents the type of service running type MentixServiceType struct { Name string `json:"Name"` } From 258fae6153056d7e4af249befd8c842918e5a4e5 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 14:16:00 +0000 Subject: [PATCH 08/17] fix lint --- pkg/metrics/driver/xcloud/xcloud.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index c1a9cfaa80..bb67bae560 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -141,6 +141,11 @@ func (d *CloudDriver) refresh() error { // send to register if catalog is set req, err = http.NewRequest("POST", d.catalog, bytes.NewBuffer(j)) + if err != nil { + log.Err(err).Msgf("xcloud: error creating POST request to: %s", d.catalog) + return err + } + resp, err = client.Do(req) if err != nil { log.Err(err).Msgf("xcloud: error registering catalog info to: %s with info: %s", d.catalog, string(j)) From 6a96bdefa31eee8b2754c460d4b0e6b52a14286f Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 14:21:15 +0000 Subject: [PATCH 09/17] fix lint --- pkg/metrics/driver/xcloud/xcloud.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index bb67bae560..1b15def165 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -20,9 +20,9 @@ package json import ( "bytes" + "crypto/tls" "encoding/json" "errors" - "crypto/tls" "fmt" "io/ioutil" "net/http" @@ -85,7 +85,7 @@ func (d *CloudDriver) refresh() error { } if resp.StatusCode != http.StatusOK { - err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", resp.StatusCode) + err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", d.instance, resp.StatusCode) log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance) return err } @@ -220,7 +220,7 @@ type CloudDataMetrics struct { TotalStorage int64 `json:"total_storage"` } -// CloudDataSettings represents the metrics gathered +// CloudDataSettings represents the metrics gathered type CloudDataSettings struct { IOPUrl string `json:"iopurl"` Sitename string `json:"sitename"` @@ -249,7 +249,7 @@ type MentixService struct { URL string `json:"URL"` } -// MentixSiteProperties represents the properties to expose the metrics endpoint +// MentixServiceProperties represents the properties to expose the metrics endpoint type MentixServiceProperties struct { MetricsPath string `json:"METRICS_PATH"` } From 45690a133dd0f6644f4da8356c31e76efe5e4b7c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 14:31:43 +0000 Subject: [PATCH 10/17] fix lint --- pkg/metrics/driver/xcloud/xcloud.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 1b15def165..adaf85bf23 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -181,7 +181,8 @@ func (d *CloudDriver) Configure(c *config.Config) error { for { select { case <-ticker.C: - d.refresh() + err := d.refresh() + log.Err(err).Msgf("xcloud: error from refresh goroutine") case <-quit: ticker.Stop() return From 6103b9e50b9fb78e8fd19733cc287f85f5c4b615 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 14:44:13 +0000 Subject: [PATCH 11/17] add changelog --- changelog/unreleased/xcloud-metrics.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/xcloud-metrics.md diff --git a/changelog/unreleased/xcloud-metrics.md b/changelog/unreleased/xcloud-metrics.md new file mode 100644 index 0000000000..6424830b78 --- /dev/null +++ b/changelog/unreleased/xcloud-metrics.md @@ -0,0 +1,7 @@ +Feature: support remote cloud gathering metrics + +The current metrics package can only gather metrics either from +json files. With this feature, the metrics can be gathered polling +the http endpoints exposed by the owncloud/nextcloud sciencemesh apps. + +https://github.com/cs3org/reva/pull/1403 From 2388fb65ab230f955b3ed9f571aabc003bea5264 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 15 Jan 2021 14:53:31 +0000 Subject: [PATCH 12/17] fix changelog --- changelog/unreleased/xcloud-metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/xcloud-metrics.md b/changelog/unreleased/xcloud-metrics.md index 6424830b78..165881677b 100644 --- a/changelog/unreleased/xcloud-metrics.md +++ b/changelog/unreleased/xcloud-metrics.md @@ -1,4 +1,4 @@ -Feature: support remote cloud gathering metrics +Enhancement: support remote cloud gathering metrics The current metrics package can only gather metrics either from json files. With this feature, the metrics can be gathered polling From 17208aa59bf7fd293a76a44fa47dcf9957e8a21e Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 18 Jan 2021 18:58:07 +0000 Subject: [PATCH 13/17] lint --- pkg/metrics/config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index f21276d01c..98ebe0070b 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -23,9 +23,9 @@ type Config struct { MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` MetricsDataLocation string `mapstructure:"metrics_data_location"` MetricsRecordInterval int `mapstructure:"metrics_record_interval"` - XcloudInstance string `mapstructure:"xcloud_instance"` - XcloudCatalog string `mapstructure:"xcloud_catalog"` - XcloudPullInterval int `mapstructure:"xcloud_pull_interval"` + XcloudInstance string `mapstructure:"xcloud_instance"` + XcloudCatalog string `mapstructure:"xcloud_catalog"` + XcloudPullInterval int `mapstructure:"xcloud_pull_interval"` } // Init sets sane defaults From f4a2292cd77019b2c8587871e12db3bd9acadc4c Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Mon, 18 Jan 2021 19:18:17 +0000 Subject: [PATCH 14/17] use new infra --- examples/metrics/xcloud.toml | 2 +- pkg/metrics/driver/xcloud/xcloud.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/metrics/xcloud.toml b/examples/metrics/xcloud.toml index 432e8d2c5b..16c8d6d646 100644 --- a/examples/metrics/xcloud.toml +++ b/examples/metrics/xcloud.toml @@ -6,7 +6,7 @@ metrics_data_driver_type = "xcloud" metrics_record_interval = 5000 xcloud_instance="http://localhost" xcloud_interval=5 -xcloud_catalog='https://sciencemesh-test.uni-muenster.de/api/mentix/?action=register' +xcloud_catalog='https://sciencemesh-test.uni-muenster.de/api/mentix/sites?action=register' [http.services.prometheus] diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index adaf85bf23..233fc65f29 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -155,6 +155,13 @@ func (d *CloudDriver) refresh() error { defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("xcloud: error registering site: status code(%d) body(%s)", resp.StatusCode, string(body)) + log.Err(err).Msg("xcloud: error registering site") + return err + } + + log.Info().Msgf("xcloud: site registered: %s", string(body)) return nil @@ -216,9 +223,9 @@ type CloudData struct { // CloudDataMetrics reprents the metrics gathered from the sciencemesh app type CloudDataMetrics struct { - TotalUsers int64 `json:"total_users"` - TotalGroups int64 `json:"total_groups"` - TotalStorage int64 `json:"total_storage"` + TotalUsers int64 `json:"numusers"` + TotalGroups int64 `json:"numgroups"` + TotalStorage int64 `json:"numstorage"` } // CloudDataSettings represents the metrics gathered From b897b220ec64d193787ee222514d850481865964 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 19 Jan 2021 08:25:09 +0000 Subject: [PATCH 15/17] fix error handling --- pkg/metrics/driver/xcloud/xcloud.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 233fc65f29..63ea207aeb 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -161,9 +161,7 @@ func (d *CloudDriver) refresh() error { return err } - log.Info().Msgf("xcloud: site registered: %s", string(body)) - return nil } @@ -189,7 +187,9 @@ func (d *CloudDriver) Configure(c *config.Config) error { select { case <-ticker.C: err := d.refresh() - log.Err(err).Msgf("xcloud: error from refresh goroutine") + if err != nil { + log.Err(err).Msgf("xcloud: error from refresh goroutine") + } case <-quit: ticker.Stop() return From 0bb2971ce83804c46aecea7dbe98e33430bbc834 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 19 Jan 2021 16:13:56 +0000 Subject: [PATCH 16/17] add config lint --- examples/metrics/xcloud.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/metrics/xcloud.toml b/examples/metrics/xcloud.toml index 16c8d6d646..95b0d1b4d8 100644 --- a/examples/metrics/xcloud.toml +++ b/examples/metrics/xcloud.toml @@ -4,9 +4,8 @@ address = "0.0.0.0:5550" [http.services.metrics] metrics_data_driver_type = "xcloud" metrics_record_interval = 5000 -xcloud_instance="http://localhost" -xcloud_interval=5 -xcloud_catalog='https://sciencemesh-test.uni-muenster.de/api/mentix/sites?action=register' +xcloud_instance = "http://localhost" +xcloud_interval = 5 +xcloud_catalog = 'https://sciencemesh-test.uni-muenster.de/api/mentix/sites?action=register' [http.services.prometheus] - From 30f1996bfe5fde49744fbd458b3823d2c3345845 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Tue, 19 Jan 2021 16:16:35 +0000 Subject: [PATCH 17/17] re-use http client from driver --- pkg/metrics/driver/xcloud/xcloud.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go index 63ea207aeb..9855b3f88d 100644 --- a/pkg/metrics/driver/xcloud/xcloud.go +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -57,17 +57,10 @@ type CloudDriver struct { pullInterval int CloudData *CloudData sync.Mutex + client *http.Client } func (d *CloudDriver) refresh() error { - // TODO(labkode): spawn goroutines to fetch metrics and update the register service - - // get configuration from internal_metrics endpoint exposed - // by the sciencemesh app - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} // endpoint example: https://mybox.com or https://mybox.com/owncloud endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance) @@ -78,7 +71,7 @@ func (d *CloudDriver) refresh() error { return err } - resp, err := client.Do(req) + resp, err := d.client.Do(req) if err != nil { log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance) return err @@ -146,7 +139,7 @@ func (d *CloudDriver) refresh() error { return err } - resp, err = client.Do(req) + resp, err = d.client.Do(req) if err != nil { log.Err(err).Msgf("xcloud: error registering catalog info to: %s with info: %s", d.catalog, string(j)) return err @@ -180,6 +173,14 @@ func (d *CloudDriver) Configure(c *config.Config) error { d.pullInterval = c.XcloudPullInterval d.catalog = c.XcloudCatalog + // TODO(labkode): make it configurable once site adopted are prod-ready + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + d.client = client + ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second) quit := make(chan struct{}) go func() {