diff --git a/changelog/unreleased/xcloud-metrics.md b/changelog/unreleased/xcloud-metrics.md new file mode 100644 index 0000000000..165881677b --- /dev/null +++ b/changelog/unreleased/xcloud-metrics.md @@ -0,0 +1,7 @@ +Enhancement: support remote cloud gathering metrics + +The current metrics package can only gather metrics either from +json files. With this feature, the metrics can be gathered polling +the http endpoints exposed by the owncloud/nextcloud sciencemesh apps. + +https://github.com/cs3org/reva/pull/1403 diff --git a/examples/metrics/xcloud.toml b/examples/metrics/xcloud.toml new file mode 100644 index 0000000000..95b0d1b4d8 --- /dev/null +++ b/examples/metrics/xcloud.toml @@ -0,0 +1,11 @@ +[http] +address = "0.0.0.0:5550" + +[http.services.metrics] +metrics_data_driver_type = "xcloud" +metrics_record_interval = 5000 +xcloud_instance = "http://localhost" +xcloud_interval = 5 +xcloud_catalog = 'https://sciencemesh-test.uni-muenster.de/api/mentix/sites?action=register' + +[http.services.prometheus] diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index 9fd7762603..424e5e1659 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -23,6 +23,9 @@ type Config struct { MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"` MetricsDataLocation string `mapstructure:"metrics_data_location"` MetricsRecordInterval int `mapstructure:"metrics_record_interval"` + XcloudInstance string `mapstructure:"xcloud_instance"` + XcloudCatalog string `mapstructure:"xcloud_catalog"` + XcloudPullInterval int `mapstructure:"xcloud_pull_interval"` } // Init sets sane defaults @@ -33,6 +36,7 @@ func (c *Config) Init() { c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json" } } + if c.MetricsRecordInterval == 0 { c.MetricsRecordInterval = 5000 } diff --git a/pkg/metrics/driver/loader/loader.go b/pkg/metrics/driver/loader/loader.go index be73f5c948..82fd0ba7f6 100644 --- a/pkg/metrics/driver/loader/loader.go +++ b/pkg/metrics/driver/loader/loader.go @@ -22,5 +22,6 @@ import ( // Load metrics drivers. _ "github.com/cs3org/reva/pkg/metrics/driver/dummy" _ "github.com/cs3org/reva/pkg/metrics/driver/json" + _ "github.com/cs3org/reva/pkg/metrics/driver/xcloud" // Add your own here ) diff --git a/pkg/metrics/driver/xcloud/xcloud.go b/pkg/metrics/driver/xcloud/xcloud.go new file mode 100644 index 0000000000..9855b3f88d --- /dev/null +++ b/pkg/metrics/driver/xcloud/xcloud.go @@ -0,0 +1,269 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package json + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "sync" + "time" + + "github.com/cs3org/reva/pkg/metrics/driver/registry" + + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/metrics/config" + "github.com/rs/zerolog" +) + +var log zerolog.Logger + +func init() { + log = logger.New().With().Int("pid", os.Getpid()).Logger() + driver := &CloudDriver{CloudData: &CloudData{}} + registry.Register(driverName(), driver) + +} + +func driverName() string { + return "xcloud" +} + +// CloudDriver is the driver to use for Sciencemesh apps +type CloudDriver struct { + instance string + catalog string + pullInterval int + CloudData *CloudData + sync.Mutex + client *http.Client +} + +func (d *CloudDriver) refresh() error { + + // endpoint example: https://mybox.com or https://mybox.com/owncloud + endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance) + + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + log.Err(err).Msgf("xcloud: error creating request to %s", d.instance) + return err + } + + resp, err := d.client.Do(req) + if err != nil { + log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance) + return err + } + + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", d.instance, resp.StatusCode) + log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance) + return err + } + defer resp.Body.Close() + + // read response body + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", d.instance) + return err + } + + cd := &CloudData{} + if err := json.Unmarshal(data, cd); err != nil { + log.Err(err).Msgf("xcloud: error parsing body from internal metrics: body(%s)", string(data)) + return err + } + + d.Lock() + defer d.Unlock() + d.CloudData = cd + log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd) + + mc := &MentixCatalog{ + Name: cd.Settings.Sitename, + FullName: cd.Settings.Sitename, + Homepage: cd.Settings.Hostname, + Description: "ScienceMesh App from " + cd.Settings.Sitename, + CountryCode: cd.Settings.Country, + Services: []*MentixService{ + &MentixService{ + Host: cd.Settings.Hostname, + IsMonitored: true, + Name: cd.Settings.Hostname + " - REVAD", + URL: cd.Settings.Siteurl, + Properties: &MentixServiceProperties{ + MetricsPath: "/index.php/apps/sciencemesh/metrics", + }, + Type: &MentixServiceType{ + Name: "REVAD", + }, + }, + }, + } + + j, err := json.Marshal(mc) + if err != nil { + log.Err(err).Msgf("xcloud: error marhsaling mentix calalog info") + return err + } + + log.Info().Msgf("xcloud: info to send to register: %s", string(j)) + + // send to register if catalog is set + req, err = http.NewRequest("POST", d.catalog, bytes.NewBuffer(j)) + if err != nil { + log.Err(err).Msgf("xcloud: error creating POST request to: %s", d.catalog) + return err + } + + resp, err = d.client.Do(req) + if err != nil { + log.Err(err).Msgf("xcloud: error registering catalog info to: %s with info: %s", d.catalog, string(j)) + return err + } + + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("xcloud: error registering site: status code(%d) body(%s)", resp.StatusCode, string(body)) + log.Err(err).Msg("xcloud: error registering site") + return err + } + + log.Info().Msgf("xcloud: site registered: %s", string(body)) + return nil +} + +// Configure configures this driver +func (d *CloudDriver) Configure(c *config.Config) error { + if c.XcloudInstance == "" { + err := errors.New("xcloud: missing xcloud_instance config parameter") + return err + } + + if c.XcloudPullInterval == 0 { + c.XcloudPullInterval = 10 // seconds + } + + d.instance = c.XcloudInstance + d.pullInterval = c.XcloudPullInterval + d.catalog = c.XcloudCatalog + + // TODO(labkode): make it configurable once site adopted are prod-ready + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + d.client = client + + ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second) + quit := make(chan struct{}) + go func() { + for { + select { + case <-ticker.C: + err := d.refresh() + if err != nil { + log.Err(err).Msgf("xcloud: error from refresh goroutine") + } + case <-quit: + ticker.Stop() + return + } + } + }() + + return nil +} + +// GetNumUsers returns the number of site users +func (d *CloudDriver) GetNumUsers() int64 { + return d.CloudData.Metrics.TotalUsers +} + +// GetNumGroups returns the number of site groups +func (d *CloudDriver) GetNumGroups() int64 { + return d.CloudData.Metrics.TotalGroups +} + +// GetAmountStorage returns the amount of site storage used +func (d *CloudDriver) GetAmountStorage() int64 { + return d.CloudData.Metrics.TotalStorage +} + +// CloudData represents the information obtained from the sciencemesh app +type CloudData struct { + Metrics CloudDataMetrics `json:"metrics"` + Settings CloudDataSettings `json:"settings"` +} + +// CloudDataMetrics reprents the metrics gathered from the sciencemesh app +type CloudDataMetrics struct { + TotalUsers int64 `json:"numusers"` + TotalGroups int64 `json:"numgroups"` + TotalStorage int64 `json:"numstorage"` +} + +// CloudDataSettings represents the metrics gathered +type CloudDataSettings struct { + IOPUrl string `json:"iopurl"` + Sitename string `json:"sitename"` + Siteurl string `json:"siteurl"` + Hostname string `json:"hostname"` + Country string `json:"country"` +} + +// MentixCatalog represents the information needed to register a site into the mesh +type MentixCatalog struct { + CountryCode string `json:"CountryCode"` + Description string `json:"Description"` + FullName string `json:"FullName"` + Homepage string `json:"Homepage"` + Name string `json:"Name"` + Services []*MentixService `json:"Services"` +} + +// MentixService represents the service running in a site +type MentixService struct { + Host string `json:"Host"` + IsMonitored bool `json:"IsMonitored"` + Name string `json:"Name"` + Properties *MentixServiceProperties `json:"Properties"` + Type *MentixServiceType `json:"Type"` + URL string `json:"URL"` +} + +// MentixServiceProperties represents the properties to expose the metrics endpoint +type MentixServiceProperties struct { + MetricsPath string `json:"METRICS_PATH"` +} + +// MentixServiceType represents the type of service running +type MentixServiceType struct { + Name string `json:"Name"` +}