From b3484014851cc72dc815022ea825d79f45f42029 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Wed, 10 May 2017 18:27:54 +0200 Subject: [PATCH 01/23] first implementation of generic sql plugin (starting from https://github.com/bbczeuz/telegraf/tree/zseng_dev_sqlquery ) --- plugins/inputs/all/all.go | 1 + plugins/inputs/sql/sql.go | 499 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 500 insertions(+) create mode 100644 plugins/inputs/sql/sql.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 10af14864f74d..ffc93410ce6ca 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -71,6 +71,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/snmp" _ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy" _ "github.com/influxdata/telegraf/plugins/inputs/socket_listener" + _ "github.com/influxdata/telegraf/plugins/inputs/sql" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" _ "github.com/influxdata/telegraf/plugins/inputs/statsd" _ "github.com/influxdata/telegraf/plugins/inputs/sysstat" diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go new file mode 100644 index 0000000000000..126d3e40f5ef8 --- /dev/null +++ b/plugins/inputs/sql/sql.go @@ -0,0 +1,499 @@ +package sql + +import ( + "bytes" + "database/sql" + "errors" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + "log" + "os" + "reflect" + "strconv" + "time" + // database drivers here: + // _ "bitbucket.org/phiggins/db2cli" // + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" // pure go + _ "github.com/mattn/go-oci8" + _ "gopkg.in/rana/ora.v4" + // _ "github.com/denisenkom/go-mssqldb" // pure go + _ "github.com/zensqlmonitor/go-mssqldb" // pure go +) + +const TYPE_STRING = 1 +const TYPE_BOOL = 2 +const TYPE_INT = 3 +const TYPE_FLOAT = 4 +const TYPE_TIME = 5 +const TYPE_AUTO = 0 + +var Debug = false + +type Query struct { + Query string + Measurement string + + TagCols []string + IntFields []string + FloatFields []string + BoolFields []string + TimeFields []string + RawFields []string + // + FieldsName []string + FieldsValue []string + // + ZeroizeNull bool + IgnoreOtherFields bool + // + QueryScript string + + // internal data + statements []*sql.Stmt + // Parameters []string + + column_name []string + cell_refs []interface{} + cells []interface{} + + field_count int + field_idx []int //Column indexes of fields + field_type []int //Column types of fields + + tag_count int + tag_idx []int //Column indexes of tags (strings) +} + +type Sql struct { + Hosts []string + + Driver string + Servers []string + KeepConnection bool + + Query []Query + + // internal + Debug bool + + connections []*sql.DB + _initialized bool +} + +//TODO +func contains_str(key string, str_array []string) bool { + for _, b := range str_array { + if b == key { + return true + } + } + return false +} + +func (s *Sql) SampleConfig() string { + var sampleConfig = ` + [[inputs.sqlquery]] + debug=false + + ## DB Driver + driver = "" # required. Options: oci8 (Oracle), postgres + # keep_connection = false # keeps the connection with database + + ## Server URLs + servers = ["telegraf/monitor@10.62.6.1:1522/tunapit"] # required. Connection URL to pass to the DB driver + hosts=["pbzasplx001.wp.lan"] + + ## Queries to perform (block below can be repeated) + [[inputs.sqlquery.query]] + query_script = "/path/to/sql/script.sql" + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + measurement="log" # destination measurement + tag_cols=["GROUP#","STATUS","NAME"] # + int_fields=["MEMBERS","FIRST_CHANGE#","BYTES","VALUE"] # + raw_fields=["UNIT"] # + time_fields=["FIRST_TIME"] # + ignore_other_fields = false # + zeroize_null = false # true: Push null results as zeros/empty strings (false: ignore fields) + + ` + return sampleConfig +} + +func (_ *Sql) Description() string { + return "SQL Plugin" +} + +func (s *Sql) initPlugin() { + Debug = s.Debug + + if Debug { + log.Printf("I! Init %d servers %d queries", len(s.Servers), len(s.Query)) + } + // s.query_cache = make([]Query, len(s.Queries)) + if s.KeepConnection { + s.connections = make([]*sql.DB, len(s.Servers)) + } + for i := 0; i < len(s.Servers); i++ { + //TODO get host from server + // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") + // fmt.Println(match) + // addr, err := net.LookupHost("198.252.206.16") + + } +} + +func (s *Query) initQuery(cols []string) error { + if Debug { + log.Printf("I! Init Query with %d columns", len(cols)) + } + s.column_name = cols + //Define index of tags and fields and keep it for reuse + col_count := len(s.column_name) + + expected_tag_count := len(s.TagCols) + var expected_field_count int + if !s.IgnoreOtherFields { + expected_field_count = col_count - expected_tag_count + } else { + expected_field_count = len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) + len(s.RawFields) + } + + if Debug { + log.Printf("I! Extpected %d tags and %d fields", expected_tag_count, expected_field_count) + } + + s.tag_idx = make([]int, expected_tag_count) //Column indexes of tags (strings) + s.field_idx = make([]int, expected_field_count) //Column indexes of fields + s.field_type = make([]int, expected_field_count) //Column types of fields + s.tag_count = 0 + s.field_count = 0 + + s.cells = make([]interface{}, col_count) + s.cell_refs = make([]interface{}, col_count) + + var cell interface{} + for i := 0; i < col_count; i++ { + if Debug { + log.Printf("I! Field %s %d", s.column_name[i], i) + } + field_matched := true + if contains_str(s.column_name[i], s.TagCols) { + field_matched = false + s.tag_idx[s.tag_count] = i + s.tag_count++ + cell = new(sql.RawBytes) + // cell = new(string); + } else if contains_str(s.column_name[i], s.IntFields) { + s.field_type[s.field_count] = TYPE_INT + cell = new(sql.RawBytes) + // cell = new(int); + } else if contains_str(s.column_name[i], s.FloatFields) { + s.field_type[s.field_count] = TYPE_FLOAT + // cell = new(float64); + cell = new(sql.RawBytes) + } else if contains_str(s.column_name[i], s.TimeFields) { + //TODO as number? + s.field_type[s.field_count] = TYPE_TIME + cell = new(string) + // cell = new(sql.RawBytes) + } else if contains_str(s.column_name[i], s.BoolFields) { + s.field_type[s.field_count] = TYPE_BOOL + // cell = new(bool); + cell = new(sql.RawBytes) + } else if contains_str(s.column_name[i], s.RawFields) { + s.field_type[s.field_count] = TYPE_AUTO + cell = new(sql.RawBytes) + } else if !s.IgnoreOtherFields { + s.field_type[s.field_count] = TYPE_AUTO + cell = new(sql.RawBytes) + // cell = new(string); + } else { + field_matched = false + cell = new(sql.RawBytes) + if Debug { + log.Printf("I! Skipped field %s", s.column_name[i]) + } + } + if field_matched { + s.field_idx[s.field_count] = i + s.field_count++ + } + s.cells[i] = cell + s.cell_refs[i] = &s.cells[i] + } + if Debug { + log.Printf("I! Query received %d tags and %d fields on %d columns...", s.tag_count, s.field_count, col_count) + } + + return nil +} + +func (s *Query) convertField(name string, cell interface{}, field_type int, NullAsZero bool) (interface{}, error) { + var value interface{} + var ok bool + var str string + var err error + + ok = true + if cell != nil { + switch field_type { + case TYPE_INT: + str, ok = cell.(string) + if ok { + value, err = strconv.ParseInt(str, 10, 64) + } + break + case TYPE_FLOAT: + str, ok = cell.(string) + if ok { + value, err = strconv.ParseFloat(str, 64) + } + break + case TYPE_BOOL: + str, ok = cell.(string) + if ok { + value, err = strconv.ParseBool(str) + } + break + // case TYPE_TIME: + // value = cell + // break + // case TYPE_STRING: + // value = cell + // break + default: + value = cell + } + + } else if NullAsZero { + value = 0 + if Debug { + log.Printf("I! forcing to 0 field name '%s' type %d", name, field_type) + } + } else { + value = nil + if Debug { + log.Printf("I! nil value for field name '%s' type %d", name, field_type) + } + } + if !ok { + cell_type := reflect.TypeOf(cell).Kind() + + log.Printf("E! converting field name '%s' type %d %s into string", name, field_type, cell_type) + err = errors.New("Error converting field into string") + return nil, err + } + if err != nil { + return nil, err + } + return value, nil +} + +func (s *Query) parseRow(query_time time.Time, host string, acc telegraf.Accumulator) error { + tags := map[string]string{} + fields := map[string]interface{}{} + + // if host != nil { + //Use database server as host, not the local host + tags["host"] = host + // } + + //Fill tags + for i := 0; i < s.tag_count; i++ { + cell := s.cells[s.tag_idx[i]] + if cell != nil { + //Tags are always strings + name := s.column_name[s.tag_idx[i]] + value, ok := cell.(string) + if !ok { + log.Printf("E! converting tag %d '%s' type %d", s.field_idx[i], name, s.field_type[i]) + return nil + } + tags[name] = value + } + } + + //Fill fields + for i := 0; i < s.field_count; i++ { + cell := s.cells[s.field_idx[i]] + name := s.column_name[s.field_idx[i]] + value, err := s.convertField(name, cell, s.field_type[i], s.ZeroizeNull) + if err != nil { + return err + } + fields[name] = value + } + + acc.AddFields(s.Measurement, fields, tags, query_time) + return nil +} + +func (p *Sql) connect(si int) (*sql.DB, error) { + var err error + + // create connection to db server if not already done + var db *sql.DB + if p.KeepConnection { + db = p.connections[si] + } else { + db = nil + } + + if db == nil { + if Debug { + log.Printf("I! Setting up DB %s %s ...", p.Driver, p.Servers[si]) + } + db, err = sql.Open(p.Driver, p.Servers[si]) + if err != nil { + return nil, err + } + } else { + if Debug { + log.Printf("I! Reusing connection to %s ...", p.Servers[si]) + } + } + + if Debug { + log.Printf("I! Connecting to DB %s ...", p.Servers[si]) + } + err = db.Ping() + if err != nil { + return nil, err + } + + if p.KeepConnection { + p.connections[si] = db + } else { + defer db.Close() + } + return db, nil +} + +func (p *Sql) Gather(acc telegraf.Accumulator) error { + if !p._initialized { + p.initPlugin() + p._initialized = true + } + + if Debug { + log.Printf("I! Starting poll") + } + for si := 0; si < len(p.Servers); si++ { + var err error + var db *sql.DB + db, err = p.connect(si) + if err != nil { + return err + } + if !p.KeepConnection { + defer db.Close() + } + + // Execute queries + for qi := 0; qi < len(p.Query); qi++ { + var rows *sql.Rows + var query_time time.Time + q := &p.Query[qi] + + // read query from sql script and put it in query string + if len(q.QueryScript) > 0 && len(q.Query) == 0 { + if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { + log.Printf("E! SQL script not exists '%s'...", q.QueryScript) + return err + } + filerc, err := os.Open(q.QueryScript) + if err != nil { + log.Fatal(err) + return err + } + defer filerc.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(filerc) + q.Query = buf.String() + if Debug { + log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, qi) + } + } + + if p.KeepConnection { + // prepare statement if not already done + if len(q.statements) == 0 { + q.statements = make([]*sql.Stmt, len(p.Servers)) + } + + if q.statements[si] == nil { + if Debug { + log.Printf("I! Preparing statement query %d...", qi) + } + q.statements[si], err = db.Prepare(q.Query) + if err != nil { + return err + } + // defer stmt.Close() + } + + // execute prepared statement + if Debug { + log.Printf("I! Performing query '%s'...", q.Query) + } + query_time = time.Now() + rows, err = q.statements[si].Query() + // err = stmt.QueryRow(1) + } else { + // execute query + if Debug { + log.Printf("I! Performing query '%s'...", q.Query) + } + query_time = time.Now() + rows, err = db.Query(q.Query) + } + + if err != nil { + return err + } + defer rows.Close() + + if q.field_count == 0 { + var cols []string + cols, err = rows.Columns() + if err != nil { + return err + } + q.initQuery(cols) + } + + row_count := 0 + + for rows.Next() { + if err = rows.Err(); err != nil { + return err + } + + err := rows.Scan(q.cell_refs...) + if err != nil { + return err + } + err = q.parseRow(query_time, p.Hosts[si], acc) + if err != nil { + return err + } + row_count += 1 + } + // if Debug { + log.Printf("I! Query %d on %s found %d rows written in %s...", qi, p.Hosts[si], row_count, q.Measurement) + // } + } + } + if Debug { + log.Printf("I! Poll done") + } + return nil +} + +func init() { + inputs.Add("sql", func() telegraf.Input { + return &Sql{} + }) +} From 9dc50809a01d0c6a40dc908728e7890a23ac847a Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Wed, 10 May 2017 19:05:43 +0200 Subject: [PATCH 02/23] camel case function names fixed initialization --- plugins/inputs/sql/sql.go | 312 ++++++++++++++++++++++++++------------ 1 file changed, 217 insertions(+), 95 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 126d3e40f5ef8..7f8ef66f3ff6c 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -4,6 +4,7 @@ import ( "bytes" "database/sql" "errors" + "github.com/gchaincl/dotsql" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "log" @@ -29,22 +30,24 @@ const TYPE_TIME = 5 const TYPE_AUTO = 0 var Debug = false +var qindex = 0 type Query struct { Query string Measurement string - TagCols []string + TagCols []string + FieldCols []string + // IntFields []string FloatFields []string BoolFields []string TimeFields []string - RawFields []string // FieldsName []string FieldsValue []string // - ZeroizeNull bool + NullAsZero bool IgnoreOtherFields bool // QueryScript string @@ -58,13 +61,33 @@ type Query struct { cells []interface{} field_count int - field_idx []int //Column indexes of fields - field_type []int //Column types of fields + field_idx []int //Column indexes of fields + field_type []int //Column types of fields tag_count int - tag_idx []int //Column indexes of tags (strings) + tag_idx []int //Column indexes of tags (strings) + + index int } +//type Database struct { +// Hosts []string +// Driver string +// Servers []string +// KeepConnection bool +// +// Query []Query +// +// // internal +// connections []*sql.DB +// _initialized bool +//} +// +//type Sql struct { +// Instance []Database +// // internal +// Debug bool +//} type Sql struct { Hosts []string @@ -93,29 +116,30 @@ func contains_str(key string, str_array []string) bool { func (s *Sql) SampleConfig() string { var sampleConfig = ` - [[inputs.sqlquery]] + [[inputs.sql]] debug=false ## DB Driver - driver = "" # required. Options: oci8 (Oracle), postgres - # keep_connection = false # keeps the connection with database + driver = "oci8" # required. Options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + # keep_connection = false # keeps the connection with database instead to reconnect at each poll ## Server URLs - servers = ["telegraf/monitor@10.62.6.1:1522/tunapit"] # required. Connection URL to pass to the DB driver - hosts=["pbzasplx001.wp.lan"] + servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection URL to pass to the DB driver + hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag ## Queries to perform (block below can be repeated) - [[inputs.sqlquery.query]] - query_script = "/path/to/sql/script.sql" + [[inputs.sql.query]] query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file measurement="log" # destination measurement - tag_cols=["GROUP#","STATUS","NAME"] # - int_fields=["MEMBERS","FIRST_CHANGE#","BYTES","VALUE"] # - raw_fields=["UNIT"] # - time_fields=["FIRST_TIME"] # - ignore_other_fields = false # - zeroize_null = false # true: Push null results as zeros/empty strings (false: ignore fields) - + tag_cols=["GROUP#","NAME"] # colums used as tags + field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + #bool_fields=["ON"] # adds fields and forces his value as bool + #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + #float_fields=["TEMPERATURE"] # adds fields and forces his value as float + #time_fields=["FIRST_TIME"] # adds fields and forces his value as time + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) + null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) ` return sampleConfig } @@ -124,15 +148,19 @@ func (_ *Sql) Description() string { return "SQL Plugin" } -func (s *Sql) initPlugin() { +func (s *Sql) Init() { Debug = s.Debug if Debug { log.Printf("I! Init %d servers %d queries", len(s.Servers), len(s.Query)) } - // s.query_cache = make([]Query, len(s.Queries)) if s.KeepConnection { s.connections = make([]*sql.DB, len(s.Servers)) + // for _, q := range s.Query { + // q.statements = make([]*sql.Stmt, len(s.Servers)) + for i := 0; i < len(s.Query); i++ { + s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) + } } for i := 0; i < len(s.Servers); i++ { //TODO get host from server @@ -143,9 +171,12 @@ func (s *Sql) initPlugin() { } } -func (s *Query) initQuery(cols []string) error { +func (s *Query) Init(cols []string) error { + qindex++ + s.index = qindex + if Debug { - log.Printf("I! Init Query with %d columns", len(cols)) + log.Printf("I! Init Query %d with %d columns", s.index, len(cols)) } s.column_name = cols //Define index of tags and fields and keep it for reuse @@ -154,18 +185,18 @@ func (s *Query) initQuery(cols []string) error { expected_tag_count := len(s.TagCols) var expected_field_count int if !s.IgnoreOtherFields { - expected_field_count = col_count - expected_tag_count + expected_field_count = col_count // - expected_tag_count } else { - expected_field_count = len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) + len(s.RawFields) + expected_field_count = len(s.FieldCols) + len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) } if Debug { log.Printf("I! Extpected %d tags and %d fields", expected_tag_count, expected_field_count) } - s.tag_idx = make([]int, expected_tag_count) //Column indexes of tags (strings) - s.field_idx = make([]int, expected_field_count) //Column indexes of fields - s.field_type = make([]int, expected_field_count) //Column types of fields + s.tag_idx = make([]int, expected_tag_count) + s.field_idx = make([]int, expected_field_count) + s.field_type = make([]int, expected_field_count) s.tag_count = 0 s.field_count = 0 @@ -201,7 +232,7 @@ func (s *Query) initQuery(cols []string) error { s.field_type[s.field_count] = TYPE_BOOL // cell = new(bool); cell = new(sql.RawBytes) - } else if contains_str(s.column_name[i], s.RawFields) { + } else if contains_str(s.column_name[i], s.FieldCols) { s.field_type[s.field_count] = TYPE_AUTO cell = new(sql.RawBytes) } else if !s.IgnoreOtherFields { @@ -229,7 +260,7 @@ func (s *Query) initQuery(cols []string) error { return nil } -func (s *Query) convertField(name string, cell interface{}, field_type int, NullAsZero bool) (interface{}, error) { +func (s *Query) ConvertField(name string, cell interface{}, field_type int, NullAsZero bool) (interface{}, error) { var value interface{} var ok bool var str string @@ -267,7 +298,15 @@ func (s *Query) convertField(name string, cell interface{}, field_type int, Null } } else if NullAsZero { - value = 0 + switch field_type { + case TYPE_AUTO: + case TYPE_STRING: + value = "" + break + default: + value = 0 + } + if Debug { log.Printf("I! forcing to 0 field name '%s' type %d", name, field_type) } @@ -290,7 +329,7 @@ func (s *Query) convertField(name string, cell interface{}, field_type int, Null return value, nil } -func (s *Query) parseRow(query_time time.Time, host string, acc telegraf.Accumulator) error { +func (s *Query) ParseRow(query_time time.Time, host string, acc telegraf.Accumulator) error { tags := map[string]string{} fields := map[string]interface{}{} @@ -318,7 +357,7 @@ func (s *Query) parseRow(query_time time.Time, host string, acc telegraf.Accumul for i := 0; i < s.field_count; i++ { cell := s.cells[s.field_idx[i]] name := s.column_name[s.field_idx[i]] - value, err := s.convertField(name, cell, s.field_type[i], s.ZeroizeNull) + value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) if err != nil { return err } @@ -329,7 +368,7 @@ func (s *Query) parseRow(query_time time.Time, host string, acc telegraf.Accumul return nil } -func (p *Sql) connect(si int) (*sql.DB, error) { +func (p *Sql) Connect(si int) (*sql.DB, error) { var err error // create connection to db server if not already done @@ -364,15 +403,81 @@ func (p *Sql) connect(si int) (*sql.DB, error) { if p.KeepConnection { p.connections[si] = db - } else { - defer db.Close() } return db, nil } +func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, error) { + var err error + var rows *sql.Rows + // read query from sql script and put it in query string + if len(q.QueryScript) > 0 && len(q.Query) == 0 { + if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { + log.Printf("E! SQL script not exists '%s'...", q.QueryScript) + return nil, err + } + filerc, err := os.Open(q.QueryScript) + if err != nil { + log.Fatal(err) + return nil, err + } + defer filerc.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(filerc) + q.Query = buf.String() + if Debug { + log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) + } + } + if len(q.Query) > 0 { + if KeepConnection { + // prepare statement if not already done + if q.statements[si] == nil { + if Debug { + log.Printf("I! Preparing statement query %d...", q.index) + } + q.statements[si], err = db.Prepare(q.Query) + if err != nil { + return nil, err + } + //defer stmt.Close() + } + + // execute prepared statement + if Debug { + log.Printf("I! Performing query '%s'...", q.Query) + } + rows, err = q.statements[si].Query() + } else { + // execute query + if Debug { + log.Printf("I! Performing query '%s'...", q.Query) + } + rows, err = db.Query(q.Query) + } + } else if len(q.QueryScript) > 0 { + // Loads queries from file + var dot *dotsql.DotSql + dot, err = dotsql.LoadFromFile(q.QueryScript) + if err != nil { + return nil, err + } + rows, err = dot.Query(db, "find-users-by-email") + } else { + log.Printf("E! No query to execute %d", q.index) + // err = errors.New("No query to execute") + // return err + return nil, nil + } + + return rows, err +} + func (p *Sql) Gather(acc telegraf.Accumulator) error { if !p._initialized { - p.initPlugin() + // if len(p.connections) == 0 { + p.Init() p._initialized = true } @@ -382,7 +487,7 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { for si := 0; si < len(p.Servers); si++ { var err error var db *sql.DB - db, err = p.connect(si) + db, err = p.Connect(si) if err != nil { return err } @@ -396,72 +501,89 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { var query_time time.Time q := &p.Query[qi] - // read query from sql script and put it in query string - if len(q.QueryScript) > 0 && len(q.Query) == 0 { - if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { - log.Printf("E! SQL script not exists '%s'...", q.QueryScript) - return err - } - filerc, err := os.Open(q.QueryScript) - if err != nil { - log.Fatal(err) - return err - } - defer filerc.Close() - - buf := new(bytes.Buffer) - buf.ReadFrom(filerc) - q.Query = buf.String() - if Debug { - log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, qi) - } - } - - if p.KeepConnection { - // prepare statement if not already done - if len(q.statements) == 0 { - q.statements = make([]*sql.Stmt, len(p.Servers)) - } - - if q.statements[si] == nil { - if Debug { - log.Printf("I! Preparing statement query %d...", qi) - } - q.statements[si], err = db.Prepare(q.Query) - if err != nil { - return err - } - // defer stmt.Close() - } - - // execute prepared statement - if Debug { - log.Printf("I! Performing query '%s'...", q.Query) - } - query_time = time.Now() - rows, err = q.statements[si].Query() - // err = stmt.QueryRow(1) - } else { - // execute query - if Debug { - log.Printf("I! Performing query '%s'...", q.Query) - } - query_time = time.Now() - rows, err = db.Query(q.Query) - } + query_time = time.Now() + rows, err = q.Execute(db, si, p.KeepConnection) + + // // read query from sql script and put it in query string + // if len(q.QueryScript) > 0 && len(q.Query) == 0 { + // if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { + // log.Printf("E! SQL script not exists '%s'...", q.QueryScript) + // return err + // } + // filerc, err := os.Open(q.QueryScript) + // if err != nil { + // log.Fatal(err) + // return err + // } + // defer filerc.Close() + // + // buf := new(bytes.Buffer) + // buf.ReadFrom(filerc) + // q.Query = buf.String() + // if Debug { + // log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) + // } + // } + // if len(q.Query) > 0 { + // if p.KeepConnection { + // // prepare statement if not already done + // if q.statements[si] == nil { + // if Debug { + // log.Printf("I! Preparing statement query %d...", q.index) + // } + // q.statements[si], err = db.Prepare(q.Query) + // if err != nil { + // return err + // } + // // defer stmt.Close() + // } + // + // // execute prepared statement + // if Debug { + // log.Printf("I! Performing query '%s'...", q.Query) + // } + // query_time = time.Now() + // rows, err = q.statements[si].Query() + // // err = stmt.QueryRow(1) + // } else { + // // execute query + // if Debug { + // log.Printf("I! Performing query '%s'...", q.Query) + // } + // query_time = time.Now() + // rows, err = db.Query(q.Query) + // } + // } else if len(q.QueryScript) > 0 { + // // Loads queries from file + // var dot *dotsql.DotSql + // dot, err = dotsql.LoadFromFile(q.QueryScript) + // if err != nil { + // return err + // } + // rows, err = dot.Query(db, "find-users-by-email") + // } else { + // log.Printf("E! No query to execute %d", q.index) + // // err = errors.New("No query to execute") + // // return err + // continue + // } if err != nil { return err } + if rows == nil { + continue + } defer rows.Close() if q.field_count == 0 { + // initialize once the structure of query var cols []string cols, err = rows.Columns() if err != nil { return err } - q.initQuery(cols) + q.Init(cols) } row_count := 0 @@ -475,14 +597,14 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { if err != nil { return err } - err = q.parseRow(query_time, p.Hosts[si], acc) + err = q.ParseRow(query_time, p.Hosts[si], acc) if err != nil { return err } row_count += 1 } // if Debug { - log.Printf("I! Query %d on %s found %d rows written in %s...", qi, p.Hosts[si], row_count, q.Measurement) + log.Printf("I! Query %d on %s found %d rows written in %s...", q.index, p.Hosts[si], row_count, q.Measurement) // } } } From 0cc87d0b244d77a3d076b645c3b643606eef438b Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 11 May 2017 12:29:15 +0200 Subject: [PATCH 03/23] added docu conversion to string of tag values tested oracle,sqlserver,mysql --- plugins/inputs/sql/README.md | 62 ++++++++ plugins/inputs/sql/sql.go | 275 ++++++++++++++++++----------------- 2 files changed, 206 insertions(+), 131 deletions(-) create mode 100644 plugins/inputs/sql/README.md diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md new file mode 100644 index 0000000000000..fa6519608510f --- /dev/null +++ b/plugins/inputs/sql/README.md @@ -0,0 +1,62 @@ +# SQL plugin + +The plugin executes simple queries or query scripts on multiple servers. +It permits to select the tags and the fields to export, if is needed fields can be forced to a choosen datatype. +Supported drivers are go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql (MySQL), pq (Postgres) +``` +``` + +## Getting started : + +First you need to grant read/select privileges on queried tables to the database user you use for the connection +For some drivers you need external shared libraries and environment variables (for instance +``` +``` + + +## Configuration: + +``` + [[inputs.sql]] + # debug=false # Enables very verbose output + + ## Database Driver + driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) + + ## Server URLs + servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection URL to pass to the DB driver + hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag + + ## Queries to perform (block below can be repeated) + [[inputs.sql.query]] + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # + measurement="log" # destination measurement + tag_cols=["GROUP#","NAME"] # colums used as tags + field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + # + #bool_fields=["ON"] # adds fields and forces his value as bool + #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + #float_fields=["TEMPERATURE"] # adds fields and forces his value as float + #time_fields=["FIRST_TIME"] # adds fields and forces his value as time + + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) + null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) + sanitize = false # true: will perform some chars substitutions (false: use value as is) + + +``` + + +## Datatypes: + + +## Example for collect multiple counters defined as COLUMNS in a table: + + +## Example for collect multiple counters defined as ROWS in a table: + + diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 7f8ef66f3ff6c..e759187ffafd1 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -4,13 +4,14 @@ import ( "bytes" "database/sql" "errors" - "github.com/gchaincl/dotsql" + // "github.com/gchaincl/dotsql" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "log" "os" "reflect" "strconv" + "strings" "time" // database drivers here: // _ "bitbucket.org/phiggins/db2cli" // @@ -44,11 +45,12 @@ type Query struct { BoolFields []string TimeFields []string // - FieldsName []string - FieldsValue []string + FieldName string + FieldValue string // NullAsZero bool IgnoreOtherFields bool + Sanitize bool // QueryScript string @@ -67,27 +69,20 @@ type Query struct { tag_count int tag_idx []int //Column indexes of tags (strings) + field_name_idx int + field_value_idx int + index int } -//type Database struct { -// Hosts []string -// Driver string -// Servers []string -// KeepConnection bool -// -// Query []Query -// -// // internal -// connections []*sql.DB -// _initialized bool -//} -// -//type Sql struct { -// Instance []Database -// // internal -// Debug bool +//func NewQuery() Query { +// something := Query{} +// something.Sanitize = true +// something.field_name_idx = -1 +// something.field_value_idx = -1 +// return something //} + type Sql struct { Hosts []string @@ -100,11 +95,26 @@ type Sql struct { // internal Debug bool - connections []*sql.DB - _initialized bool + connections []*sql.DB + initialized bool +} + +var sanitizedChars = strings.NewReplacer("/sec", "_persec", "/Sec", "_persec", + " ", "_", "%", "Percent", `\`, "") + +func trimSuffix(s, suffix string) string { + for strings.HasSuffix(s, suffix) { + s = s[:len(s)-len(suffix)] + } + return s +} + +func sanitize(text string) string { + text = sanitizedChars.Replace(text) + text = trimSuffix(text, "_") + return text } -//TODO func contains_str(key string, str_array []string) bool { for _, b := range str_array { if b == key { @@ -117,11 +127,11 @@ func contains_str(key string, str_array []string) bool { func (s *Sql) SampleConfig() string { var sampleConfig = ` [[inputs.sql]] - debug=false + # debug=false # Enables very verbose output - ## DB Driver - driver = "oci8" # required. Options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) - # keep_connection = false # keeps the connection with database instead to reconnect at each poll + ## Database Driver + driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) ## Server URLs servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection URL to pass to the DB driver @@ -129,17 +139,22 @@ func (s *Sql) SampleConfig() string { ## Queries to perform (block below can be repeated) [[inputs.sql.query]] - query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" - query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # measurement="log" # destination measurement tag_cols=["GROUP#","NAME"] # colums used as tags field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + # #bool_fields=["ON"] # adds fields and forces his value as bool #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer #float_fields=["TEMPERATURE"] # adds fields and forces his value as float #time_fields=["FIRST_TIME"] # adds fields and forces his value as time + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) + sanitize = false # true: will perform some chars substitutions (false: use value as is) ` return sampleConfig } @@ -203,12 +218,43 @@ func (s *Query) Init(cols []string) error { s.cells = make([]interface{}, col_count) s.cell_refs = make([]interface{}, col_count) + s.field_name_idx = -1 + s.field_value_idx = -1 + if len(s.FieldName) > 0 && !contains_str(s.FieldName, s.column_name) { + log.Printf("E! Missing given field_name in columns: %s", s.FieldName) + err := errors.New("Missing given field_name in columns") + return err + } + if len(s.FieldValue) > 0 && !contains_str(s.FieldValue, s.column_name) { + log.Printf("E! Missing given field_value in columns: %s", s.FieldValue) + err := errors.New("Missing given field_value in columns") + return err + } + if (len(s.FieldValue) > 0 && len(s.FieldName) == 0) || (len(s.FieldName) > 0 && len(s.FieldValue) == 0) { + err := errors.New("E! Both field_name and field_value should be set") + return err + } + // if len(s.FieldName) > 0 && len(s.FieldValue) > 0 { + // s.column_name = make([]string, len(s.TagCols)+2) + // s.column_name = s.TagCols + // s.column_name = append(s.column_name, s.FieldName) + // s.column_name = append(s.column_name, s.FieldValue) + // } + var cell interface{} for i := 0; i < col_count; i++ { if Debug { log.Printf("I! Field %s %d", s.column_name[i], i) } field_matched := true + + if s.column_name[i] == s.FieldName { + s.field_name_idx = i + } + if s.column_name[i] == s.FieldValue { + s.field_value_idx = i + } + if contains_str(s.column_name[i], s.TagCols) { field_matched = false s.tag_idx[s.tag_count] = i @@ -329,42 +375,58 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null return value, nil } -func (s *Query) ParseRow(query_time time.Time, host string, acc telegraf.Accumulator) error { - tags := map[string]string{} - fields := map[string]interface{}{} - - // if host != nil { - //Use database server as host, not the local host - tags["host"] = host - // } - - //Fill tags +func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}) error { + // fill tags for i := 0; i < s.tag_count; i++ { cell := s.cells[s.tag_idx[i]] if cell != nil { - //Tags are always strings + // tags are always strings name := s.column_name[s.tag_idx[i]] value, ok := cell.(string) if !ok { log.Printf("E! converting tag %d '%s' type %d", s.field_idx[i], name, s.field_type[i]) return nil } - tags[name] = value + if s.Sanitize { + tags[name] = sanitize(value) + } else { + tags[name] = value + } } } - //Fill fields - for i := 0; i < s.field_count; i++ { - cell := s.cells[s.field_idx[i]] - name := s.column_name[s.field_idx[i]] - value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) + if s.field_name_idx >= 0 { + // get the name of the field from value on column + cell := s.cells[s.field_name_idx] + name, ok := cell.(string) + if !ok { + log.Printf("E! converting field name idx %d '%s'", s.field_name_idx, name, s.column_name[s.field_name_idx]) + return nil + } + + if s.Sanitize { + name = sanitize(name) + } + + // get the value of field + cell = s.cells[s.field_value_idx] + value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, TYPE_AUTO, s.NullAsZero) // TODO set forced field type if err != nil { return err } fields[name] = value + } else { + // fill fields from column values + for i := 0; i < s.field_count; i++ { + cell := s.cells[s.field_idx[i]] + name := s.column_name[s.field_idx[i]] + value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) + if err != nil { + return err + } + fields[name] = value + } } - - acc.AddFields(s.Measurement, fields, tags, query_time) return nil } @@ -456,18 +518,18 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } rows, err = db.Query(q.Query) } - } else if len(q.QueryScript) > 0 { - // Loads queries from file - var dot *dotsql.DotSql - dot, err = dotsql.LoadFromFile(q.QueryScript) - if err != nil { - return nil, err - } - rows, err = dot.Query(db, "find-users-by-email") + // } else if len(q.QueryScript) > 0 { + // // Loads queries from file + // var dot *dotsql.DotSql + // dot, err = dotsql.LoadFromFile(q.QueryScript) + // if err != nil { + // return nil, err + // } + // rows, err = dot.Query(db, "find-users-by-email") } else { - log.Printf("E! No query to execute %d", q.index) + log.Printf("W! No query to execute %d", q.index) // err = errors.New("No query to execute") - // return err + // return nil, err return nil, nil } @@ -475,10 +537,9 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } func (p *Sql) Gather(acc telegraf.Accumulator) error { - if !p._initialized { - // if len(p.connections) == 0 { + if !p.initialized { p.Init() - p._initialized = true + p.initialized = true } if Debug { @@ -495,7 +556,7 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { defer db.Close() } - // Execute queries + // execute queries for qi := 0; qi < len(p.Query); qi++ { var rows *sql.Rows var query_time time.Time @@ -503,70 +564,12 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { query_time = time.Now() rows, err = q.Execute(db, si, p.KeepConnection) - - // // read query from sql script and put it in query string - // if len(q.QueryScript) > 0 && len(q.Query) == 0 { - // if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { - // log.Printf("E! SQL script not exists '%s'...", q.QueryScript) - // return err - // } - // filerc, err := os.Open(q.QueryScript) - // if err != nil { - // log.Fatal(err) - // return err - // } - // defer filerc.Close() - // - // buf := new(bytes.Buffer) - // buf.ReadFrom(filerc) - // q.Query = buf.String() - // if Debug { - // log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) - // } - // } - // if len(q.Query) > 0 { - // if p.KeepConnection { - // // prepare statement if not already done - // if q.statements[si] == nil { - // if Debug { - // log.Printf("I! Preparing statement query %d...", q.index) - // } - // q.statements[si], err = db.Prepare(q.Query) - // if err != nil { - // return err - // } - // // defer stmt.Close() - // } - // - // // execute prepared statement - // if Debug { - // log.Printf("I! Performing query '%s'...", q.Query) - // } - // query_time = time.Now() - // rows, err = q.statements[si].Query() - // // err = stmt.QueryRow(1) - // } else { - // // execute query - // if Debug { - // log.Printf("I! Performing query '%s'...", q.Query) - // } - // query_time = time.Now() - // rows, err = db.Query(q.Query) - // } - // } else if len(q.QueryScript) > 0 { - // // Loads queries from file - // var dot *dotsql.DotSql - // dot, err = dotsql.LoadFromFile(q.QueryScript) - // if err != nil { - // return err - // } - // rows, err = dot.Query(db, "find-users-by-email") - // } else { - // log.Printf("E! No query to execute %d", q.index) - // // err = errors.New("No query to execute") - // // return err - // continue - // } + if Debug { + duration := time.Since(query_time) + // delta := time.Now().Sub(a) + log.Printf("I! Query %d exectution time: %s", q.index, duration) + } + query_time = time.Now() if err != nil { return err @@ -592,20 +595,30 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { if err = rows.Err(); err != nil { return err } - + // database driver datatype conversion err := rows.Scan(q.cell_refs...) if err != nil { return err } - err = q.ParseRow(query_time, p.Hosts[si], acc) + // collect tags and fields + tags := map[string]string{} + fields := map[string]interface{}{} + + // use database server as host, not the local host + tags["host"] = p.Hosts[si] + + err = q.ParseRow(tags, fields) + acc.AddFields(q.Measurement, fields, tags, query_time) + + // err = q.ParseRow(query_time, p.Hosts[si], acc) if err != nil { return err } row_count += 1 } - // if Debug { - log.Printf("I! Query %d on %s found %d rows written in %s...", q.index, p.Hosts[si], row_count, q.Measurement) - // } + if Debug { + log.Printf("I! Query %d on %s found %d rows written in %s...", q.index, p.Hosts[si], row_count, q.Measurement) + } } } if Debug { From c2703373b0f2655177e4c491592432a94bed8cb5 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 11 May 2017 12:29:15 +0200 Subject: [PATCH 04/23] added docu conversion to string of tag values tested oracle,sqlserver,mysql --- plugins/inputs/sql/README.md | 121 ++++++++++++++ plugins/inputs/sql/sql.go | 310 +++++++++++++++++++---------------- 2 files changed, 289 insertions(+), 142 deletions(-) create mode 100644 plugins/inputs/sql/README.md diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md new file mode 100644 index 0000000000000..b8fb970ee33d6 --- /dev/null +++ b/plugins/inputs/sql/README.md @@ -0,0 +1,121 @@ +# SQL plugin + +The plugin executes simple queries or query scripts on multiple servers. +It permits to select the tags and the fields to export, if is needed fields can be forced to a choosen datatype. +Supported drivers are go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql (MySQL), pq (Postgres) +``` +``` + +## Getting started : + +First you need to grant read/select privileges on queried tables to the database user you use for the connection +For some not pure go drivers you may need external shared libraries and environment variables: look at sql driver implementation site +``` +``` + + +## Configuration: + +``` + [[inputs.sql]] + # debug=false # Enables very verbose output + + ## Database Driver + driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) + + ## Server DSNs + servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection DSN to pass to the DB driver + hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag + + ## Queries to perform (block below can be repeated) + [[inputs.sql.query]] + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # + measurement="log" # destination measurement + tag_cols=["GROUP#","NAME"] # colums used as tags + field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + # + #bool_fields=["ON"] # adds fields and forces his value as bool + #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + #float_fields=["TEMPERATURE"] # adds fields and forces his value as float + #time_fields=["FIRST_TIME"] # adds fields and forces his value as time + + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) + null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) + sanitize = false # true: will perform some chars substitutions (false: use value as is) + + +``` + + +## Datatypes: + + +## Example for collect multiple counters defined as COLUMNS in a table: +Here we read a table where each counter is on a different row. Each row contains a column with the name of the counter (counter_name) and a column with his value (cntr_value) and some other columns that we use as tags (instance_name,object_name) + +``` +[[inputs.sql]] + interval = "60s" + driver = "mssql" + servers = [ + "Server=mssqlserver1.my.lan;Port=1433;User Id=telegraf;Password=secret;app name=telegraf" + "Server=mssqlserver2.my.lan;Port=1433;User Id=telegraf;Password=secret;app name=telegraf" + ] + hosts=["mssqlserver_cluster_1","mssqlserver_cluster_2"] + + [[inputs.sql.query]] + measurement = "os_performance_counters" + ignore_other_fields=true + sanitize=true + query="SELECT * FROM sys.dm_os_performance_counters WHERE object_name NOT LIKE '%Deprecated%' ORDER BY counter_name" + tag_cols=["instance_name","object_name"] + field_name = "counter_name" + field_value = "cntr_value" +``` +### Result: +``` +> os_performance_counters,host=mssqlserver_cluster_1,object_name=MSSQL$TESTSQL2014:Broker_Statistics Activation_Errors_Total=0i 1494496261000000000 +> os_performance_counters,host=mssqlserver_cluster_1,object_name=MSSQL$TESTSQL2014:Cursor_Manager_by_Type,instance_name=TSQL_Local_Cursor Active_cursors=0i 1494496261000000000 +> os_performance_counters,instance_name=TSQL_Global_Cursor,host=mssqlserver_cluster_1,object_name=MSSQL$TESTSQL2014:Cursor_Manager_by_Type Active_cursors=0i 1494496261000000000 +> os_performance_counters,host=mssqlserver_cluster_1,object_name=MSSQL$TESTSQL2014:Cursor_Manager_by_Type,instance_name=API_Cursor Active_cursors=0i 1494496261000000000 +> os_performance_counters,host=mssqlserver_cluster_1,object_name=MSSQL$TESTSQL2014:Cursor_Manager_by_Type,instance_name=_Total Active_cursors=0i 1494496261000000000 +... + +``` +## Example for collect multiple counters defined as ROWS in a table: +Here we read multiple counters defined on same row where the counter name is the name of his column. +In this example we force some counters datatypes: "MEMBERS","FIRST_CHANGE#" as integer, "BYTES" as float, "FIRST_TIME" as time. The field "UNIT" is used with the automatic driver datatype conversion. +The column "ARCHIVED" is ignored + +``` +[[inputs.sql]] + interval = "20s" + + driver = "oci8" + keep_connection=true + servers = ["telegraf/monitor@10.62.6.1:1522/tunapit"] + hosts=["oraclehost.my.lan"] + ## Queries to perform + [[inputs.sql.query]] + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + measurement="log" + tag_cols=["GROUP#","STATUS","NAME"] + field_cols=["UNIT"] + int_fields=["MEMBERS","FIRST_CHANGE#"] + float_fields=["BYTES"] + time_fields=["FIRST_TIME"] + ignore_other_fields=true +``` +### Result: +``` +> log,host=pbzasplx001.wp.lan,GROUP#=1,STATUS=INACTIVE MEMBERS=1i,FIRST_TIME="2017-05-10 22:08:38 +0200 CEST",FIRST_CHANGE#=368234811i,BYTES=52428800 1494496874000000000 +> log,host=pbzasplx001.wp.lan,GROUP#=2,STATUS=CURRENT MEMBERS=1i,FIRST_TIME="2017-05-10 22:08:38 +0200 CEST",FIRST_CHANGE#=368234816i,BYTES=52428800 1494496874000000000 +> log,host=pbzasplx001.wp.lan,GROUP#=3,STATUS=INACTIVE MEMBERS=1i,FIRST_TIME="2017-05-10 16:00:55 +0200 CEST",FIRST_CHANGE#=368220858i,BYTES=52428800 1494496874000000000 + + +``` + diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 7f8ef66f3ff6c..dcc31bb20108c 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -4,13 +4,15 @@ import ( "bytes" "database/sql" "errors" - "github.com/gchaincl/dotsql" + // "github.com/gchaincl/dotsql" + "fmt" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "log" "os" "reflect" "strconv" + "strings" "time" // database drivers here: // _ "bitbucket.org/phiggins/db2cli" // @@ -44,11 +46,12 @@ type Query struct { BoolFields []string TimeFields []string // - FieldsName []string - FieldsValue []string + FieldName string + FieldValue string // NullAsZero bool IgnoreOtherFields bool + Sanitize bool // QueryScript string @@ -67,27 +70,12 @@ type Query struct { tag_count int tag_idx []int //Column indexes of tags (strings) + field_name_idx int + field_value_idx int + index int } -//type Database struct { -// Hosts []string -// Driver string -// Servers []string -// KeepConnection bool -// -// Query []Query -// -// // internal -// connections []*sql.DB -// _initialized bool -//} -// -//type Sql struct { -// Instance []Database -// // internal -// Debug bool -//} type Sql struct { Hosts []string @@ -100,11 +88,26 @@ type Sql struct { // internal Debug bool - connections []*sql.DB - _initialized bool + connections []*sql.DB + initialized bool +} + +var sanitizedChars = strings.NewReplacer("/sec", "_persec", "/Sec", "_persec", + " ", "_", "%", "Percent", `\`, "") + +func trimSuffix(s, suffix string) string { + for strings.HasSuffix(s, suffix) { + s = s[:len(s)-len(suffix)] + } + return s +} + +func sanitize(text string) string { + text = sanitizedChars.Replace(text) + text = trimSuffix(text, "_") + return text } -//TODO func contains_str(key string, str_array []string) bool { for _, b := range str_array { if b == key { @@ -117,29 +120,34 @@ func contains_str(key string, str_array []string) bool { func (s *Sql) SampleConfig() string { var sampleConfig = ` [[inputs.sql]] - debug=false + # debug=false # Enables very verbose output - ## DB Driver - driver = "oci8" # required. Options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) - # keep_connection = false # keeps the connection with database instead to reconnect at each poll + ## Database Driver + driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - ## Server URLs - servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection URL to pass to the DB driver + ## Server DSNs + servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection DSN to pass to the DB driver hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag ## Queries to perform (block below can be repeated) [[inputs.sql.query]] - query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" - query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # measurement="log" # destination measurement tag_cols=["GROUP#","NAME"] # colums used as tags field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + # #bool_fields=["ON"] # adds fields and forces his value as bool #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer #float_fields=["TEMPERATURE"] # adds fields and forces his value as float #time_fields=["FIRST_TIME"] # adds fields and forces his value as time + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) + sanitize = false # true: will perform some chars substitutions (false: use value as is) ` return sampleConfig } @@ -148,7 +156,7 @@ func (_ *Sql) Description() string { return "SQL Plugin" } -func (s *Sql) Init() { +func (s *Sql) Init() error { Debug = s.Debug if Debug { @@ -169,6 +177,11 @@ func (s *Sql) Init() { // addr, err := net.LookupHost("198.252.206.16") } + if len(s.Servers) > 0 && len(s.Servers) != len(s.Hosts) { + return errors.New("For each server a host should be specified") + + } + return nil } func (s *Query) Init(cols []string) error { @@ -203,12 +216,41 @@ func (s *Query) Init(cols []string) error { s.cells = make([]interface{}, col_count) s.cell_refs = make([]interface{}, col_count) + s.field_name_idx = -1 + s.field_value_idx = -1 + if len(s.FieldName) > 0 && !contains_str(s.FieldName, s.column_name) { + log.Printf("E! Missing given field_name in columns: %s", s.FieldName) + err := errors.New("Missing given field_name in columns") + return err + } + if len(s.FieldValue) > 0 && !contains_str(s.FieldValue, s.column_name) { + log.Printf("E! Missing given field_value in columns: %s", s.FieldValue) + return errors.New("Missing given field_value in columns") + } + if (len(s.FieldValue) > 0 && len(s.FieldName) == 0) || (len(s.FieldName) > 0 && len(s.FieldValue) == 0) { + return errors.New("Both field_name and field_value should be set") + } + // if len(s.FieldName) > 0 && len(s.FieldValue) > 0 { + // s.column_name = make([]string, len(s.TagCols)+2) + // s.column_name = s.TagCols + // s.column_name = append(s.column_name, s.FieldName) + // s.column_name = append(s.column_name, s.FieldValue) + // } + var cell interface{} for i := 0; i < col_count; i++ { if Debug { log.Printf("I! Field %s %d", s.column_name[i], i) } field_matched := true + + if s.column_name[i] == s.FieldName { + s.field_name_idx = i + } + if s.column_name[i] == s.FieldValue { + s.field_value_idx = i + } + if contains_str(s.column_name[i], s.TagCols) { field_matched = false s.tag_idx[s.tag_count] = i @@ -329,42 +371,63 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null return value, nil } -func (s *Query) ParseRow(query_time time.Time, host string, acc telegraf.Accumulator) error { - tags := map[string]string{} - fields := map[string]interface{}{} - - // if host != nil { - //Use database server as host, not the local host - tags["host"] = host - // } - - //Fill tags +func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}) error { + // fill tags for i := 0; i < s.tag_count; i++ { cell := s.cells[s.tag_idx[i]] if cell != nil { - //Tags are always strings + // tags are always strings name := s.column_name[s.tag_idx[i]] - value, ok := cell.(string) - if !ok { - log.Printf("E! converting tag %d '%s' type %d", s.field_idx[i], name, s.field_type[i]) - return nil + + //TODO set flag for force tag data conversion? +// value, ok := cell.(string) +// if !ok { +// log.Printf("W! converting tag %d '%s' type %d", s.tag_idx[i], name, TYPE_STRING) +// return nil +// // return errors.New("Cannot convert tag") +// } + value := fmt.Sprintf("%v", cell) + if s.Sanitize { + tags[name] = sanitize(value) + } else { + tags[name] = value } - tags[name] = value } } - //Fill fields - for i := 0; i < s.field_count; i++ { - cell := s.cells[s.field_idx[i]] - name := s.column_name[s.field_idx[i]] - value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) + if s.field_name_idx >= 0 { + // get the name of the field from value on column + cell := s.cells[s.field_name_idx] + name, ok := cell.(string) + if !ok { + log.Printf("W! converting field name idx %d '%s'", s.field_name_idx, name, s.column_name[s.field_name_idx]) + return nil + // return errors.New("Cannot convert tag") + } + + if s.Sanitize { + name = sanitize(name) + } + + // get the value of field + cell = s.cells[s.field_value_idx] + value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, TYPE_AUTO, s.NullAsZero) // TODO set forced field type if err != nil { return err } fields[name] = value + } else { + // fill fields from column values + for i := 0; i < s.field_count; i++ { + cell := s.cells[s.field_idx[i]] + name := s.column_name[s.field_idx[i]] + value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) + if err != nil { + return err + } + fields[name] = value + } } - - acc.AddFields(s.Measurement, fields, tags, query_time) return nil } @@ -456,18 +519,18 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } rows, err = db.Query(q.Query) } - } else if len(q.QueryScript) > 0 { - // Loads queries from file - var dot *dotsql.DotSql - dot, err = dotsql.LoadFromFile(q.QueryScript) - if err != nil { - return nil, err - } - rows, err = dot.Query(db, "find-users-by-email") + // } else if len(q.QueryScript) > 0 { + // // Loads queries from file + // var dot *dotsql.DotSql + // dot, err = dotsql.LoadFromFile(q.QueryScript) + // if err != nil { + // return nil, err + // } + // rows, err = dot.Query(db, "find-users-by-email") } else { - log.Printf("E! No query to execute %d", q.index) + log.Printf("W! No query to execute %d", q.index) // err = errors.New("No query to execute") - // return err + // return nil, err return nil, nil } @@ -475,19 +538,30 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } func (p *Sql) Gather(acc telegraf.Accumulator) error { - if !p._initialized { - // if len(p.connections) == 0 { - p.Init() - p._initialized = true + var err error + + if !p.initialized { + err = p.Init() + if err != nil { + return err + } + p.initialized = true } if Debug { log.Printf("I! Starting poll") } for si := 0; si < len(p.Servers); si++ { - var err error var db *sql.DB + var query_time time.Time + db, err = p.Connect(si) + query_time = time.Now() + if Debug { + duration := time.Since(query_time) + log.Printf("I! Server %d connection time: %s", si, duration) + } + if err != nil { return err } @@ -495,78 +569,19 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { defer db.Close() } - // Execute queries + // execute queries for qi := 0; qi < len(p.Query); qi++ { var rows *sql.Rows - var query_time time.Time q := &p.Query[qi] query_time = time.Now() rows, err = q.Execute(db, si, p.KeepConnection) - - // // read query from sql script and put it in query string - // if len(q.QueryScript) > 0 && len(q.Query) == 0 { - // if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { - // log.Printf("E! SQL script not exists '%s'...", q.QueryScript) - // return err - // } - // filerc, err := os.Open(q.QueryScript) - // if err != nil { - // log.Fatal(err) - // return err - // } - // defer filerc.Close() - // - // buf := new(bytes.Buffer) - // buf.ReadFrom(filerc) - // q.Query = buf.String() - // if Debug { - // log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) - // } - // } - // if len(q.Query) > 0 { - // if p.KeepConnection { - // // prepare statement if not already done - // if q.statements[si] == nil { - // if Debug { - // log.Printf("I! Preparing statement query %d...", q.index) - // } - // q.statements[si], err = db.Prepare(q.Query) - // if err != nil { - // return err - // } - // // defer stmt.Close() - // } - // - // // execute prepared statement - // if Debug { - // log.Printf("I! Performing query '%s'...", q.Query) - // } - // query_time = time.Now() - // rows, err = q.statements[si].Query() - // // err = stmt.QueryRow(1) - // } else { - // // execute query - // if Debug { - // log.Printf("I! Performing query '%s'...", q.Query) - // } - // query_time = time.Now() - // rows, err = db.Query(q.Query) - // } - // } else if len(q.QueryScript) > 0 { - // // Loads queries from file - // var dot *dotsql.DotSql - // dot, err = dotsql.LoadFromFile(q.QueryScript) - // if err != nil { - // return err - // } - // rows, err = dot.Query(db, "find-users-by-email") - // } else { - // log.Printf("E! No query to execute %d", q.index) - // // err = errors.New("No query to execute") - // // return err - // continue - // } + if Debug { + duration := time.Since(query_time) + // delta := time.Now().Sub(a) + log.Printf("I! Query %d exectution time: %s", q.index, duration) + } + query_time = time.Now() if err != nil { return err @@ -592,20 +607,31 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { if err = rows.Err(); err != nil { return err } - + // database driver datatype conversion err := rows.Scan(q.cell_refs...) if err != nil { return err } - err = q.ParseRow(query_time, p.Hosts[si], acc) + // collect tags and fields + tags := map[string]string{} + fields := map[string]interface{}{} + + // use database server as host, not the local host + tags["host"] = p.Hosts[si] + + err = q.ParseRow(tags, fields) + acc.AddFields(q.Measurement, fields, tags, query_time) + + // err = q.ParseRow(query_time, p.Hosts[si], acc) if err != nil { return err } row_count += 1 } - // if Debug { - log.Printf("I! Query %d on %s found %d rows written in %s...", q.index, p.Hosts[si], row_count, q.Measurement) - // } + if Debug { + duration := time.Since(query_time) + log.Printf("I! Query %d on %s found %d rows written in %s... processing duration %s", q.index, p.Hosts[si], row_count, q.Measurement, duration) + } } } if Debug { From b8e0736b62c27f35cef56b0e379e35751f1a65b6 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 11 May 2017 15:56:44 +0200 Subject: [PATCH 05/23] added first draft for get timestamp from row updated README input list --- README.md | 1 + plugins/inputs/sql/README.md | 14 +++-- plugins/inputs/sql/sql.go | 112 +++++++++++++++++++++++------------ 3 files changed, 85 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 2c7d40b28139e..6f81dc3ded7c2 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ configuration options. * [sensors](./plugins/inputs/sensors) * [snmp](./plugins/inputs/snmp) * [snmp_legacy](./plugins/inputs/snmp_legacy) +* [sql](./plugins/inputs/sql) (sql generic) * [sql server](./plugins/inputs/sqlserver) (microsoft) * [twemproxy](./plugins/inputs/twemproxy) * [varnish](./plugins/inputs/varnish) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index b8fb970ee33d6..9c6049b068969 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -17,6 +17,7 @@ For some not pure go drivers you may need external shared libraries and environm ## Configuration: ``` + [[inputs.sql]] # debug=false # Enables very verbose output @@ -38,10 +39,15 @@ For some not pure go drivers you may need external shared libraries and environm tag_cols=["GROUP#","NAME"] # colums used as tags field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion # - #bool_fields=["ON"] # adds fields and forces his value as bool - #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer - #float_fields=["TEMPERATURE"] # adds fields and forces his value as float - #time_fields=["FIRST_TIME"] # adds fields and forces his value as time + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + # float_fields=["TEMPERATURE"] # adds fields and forces his value as float + # time_fields=["FIRST_TIME"] # adds fields and forces his value as time + # + # field_name = "counter_name" # the column that contains the name of the counter + # field_value = "counter_value" # the column that contains the value of the counter + # + # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index dcc31bb20108c..36ce7248f93a0 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -6,14 +6,15 @@ import ( "errors" // "github.com/gchaincl/dotsql" "fmt" - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/inputs" "log" "os" "reflect" "strconv" "strings" "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" // database drivers here: // _ "bitbucket.org/phiggins/db2cli" // _ "github.com/go-sql-driver/mysql" @@ -37,7 +38,10 @@ var qindex = 0 type Query struct { Query string Measurement string - + // + FieldTimestamp string + TimestampUnit string + // TagCols []string FieldCols []string // @@ -70,8 +74,9 @@ type Query struct { tag_count int tag_idx []int //Column indexes of tags (strings) - field_name_idx int - field_value_idx int + field_name_idx int + field_value_idx int + field_timestamp_idx int index int } @@ -121,30 +126,35 @@ func (s *Sql) SampleConfig() string { var sampleConfig = ` [[inputs.sql]] # debug=false # Enables very verbose output - + ## Database Driver driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - + ## Server DSNs servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection DSN to pass to the DB driver hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag - + ## Queries to perform (block below can be repeated) [[inputs.sql.query]] # query has precedence on query_script, if both query and query_script are defined only query is executed - query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file # measurement="log" # destination measurement tag_cols=["GROUP#","NAME"] # colums used as tags field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion # - #bool_fields=["ON"] # adds fields and forces his value as bool - #int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer - #float_fields=["TEMPERATURE"] # adds fields and forces his value as float - #time_fields=["FIRST_TIME"] # adds fields and forces his value as time - + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + # float_fields=["TEMPERATURE"] # adds fields and forces his value as float + # time_fields=["FIRST_TIME"] # adds fields and forces his value as time + # + # field_name = "counter_name" # the column that contains the name of the counter + # field_value = "counter_value" # the column that contains the value of the counter + # + # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) sanitize = false # true: will perform some chars substitutions (false: use value as is) @@ -218,6 +228,8 @@ func (s *Query) Init(cols []string) error { s.field_name_idx = -1 s.field_value_idx = -1 + s.field_timestamp_idx = -1 + if len(s.FieldName) > 0 && !contains_str(s.FieldName, s.column_name) { log.Printf("E! Missing given field_name in columns: %s", s.FieldName) err := errors.New("Missing given field_name in columns") @@ -230,12 +242,6 @@ func (s *Query) Init(cols []string) error { if (len(s.FieldValue) > 0 && len(s.FieldName) == 0) || (len(s.FieldName) > 0 && len(s.FieldValue) == 0) { return errors.New("Both field_name and field_value should be set") } - // if len(s.FieldName) > 0 && len(s.FieldValue) > 0 { - // s.column_name = make([]string, len(s.TagCols)+2) - // s.column_name = s.TagCols - // s.column_name = append(s.column_name, s.FieldName) - // s.column_name = append(s.column_name, s.FieldValue) - // } var cell interface{} for i := 0; i < col_count; i++ { @@ -249,6 +255,11 @@ func (s *Query) Init(cols []string) error { } if s.column_name[i] == s.FieldValue { s.field_value_idx = i + // TODO force datatype? + } + if s.column_name[i] == s.FieldTimestamp { + s.field_timestamp_idx = i + // TODO force datatype? } if contains_str(s.column_name[i], s.TagCols) { @@ -371,21 +382,45 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null return value, nil } -func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}) error { +func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, timestamp time.Time) (time.Time, error) { + if s.field_timestamp_idx >= 0 { + var ok bool + // get the value of timestamp field + cell := s.cells[s.field_timestamp_idx] + value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], cell, TYPE_AUTO, s.NullAsZero) + if err != nil { + return timestamp, err + } + timestamp, ok = value.(time.Time) + // TODO convert to ns/ms/s/us?? + if !ok { + log.Printf("W! Unable to convert timestamp '%s' to time.Time", s.column_name[s.field_timestamp_idx]) + var intvalue int64 + intvalue, ok = value.(int64) + if !ok { + cell_type := reflect.TypeOf(cell).Kind() + + log.Printf("E! Unable to convert timestamp '%s' type %s", s.column_name[s.field_timestamp_idx], cell_type) + return timestamp, errors.New("Cannot convert timestamp") + } + timestamp = time.Unix(intvalue, 0) + } + } + // fill tags for i := 0; i < s.tag_count; i++ { cell := s.cells[s.tag_idx[i]] if cell != nil { // tags are always strings name := s.column_name[s.tag_idx[i]] - + //TODO set flag for force tag data conversion? -// value, ok := cell.(string) -// if !ok { -// log.Printf("W! converting tag %d '%s' type %d", s.tag_idx[i], name, TYPE_STRING) -// return nil -// // return errors.New("Cannot convert tag") -// } + // value, ok := cell.(string) + // if !ok { + // log.Printf("W! converting tag %d '%s' type %d", s.tag_idx[i], name, TYPE_STRING) + // return nil + // // return errors.New("Cannot convert tag") + // } value := fmt.Sprintf("%v", cell) if s.Sanitize { tags[name] = sanitize(value) @@ -400,8 +435,8 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}) cell := s.cells[s.field_name_idx] name, ok := cell.(string) if !ok { - log.Printf("W! converting field name idx %d '%s'", s.field_name_idx, name, s.column_name[s.field_name_idx]) - return nil + log.Printf("W! converting field name '%s'", s.column_name[s.field_name_idx]) + return timestamp, nil // return errors.New("Cannot convert tag") } @@ -413,7 +448,7 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}) cell = s.cells[s.field_value_idx] value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, TYPE_AUTO, s.NullAsZero) // TODO set forced field type if err != nil { - return err + return timestamp, err } fields[name] = value } else { @@ -423,12 +458,12 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}) name := s.column_name[s.field_idx[i]] value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) if err != nil { - return err + return timestamp, err } fields[name] = value } } - return nil + return timestamp, nil } func (p *Sql) Connect(si int) (*sql.DB, error) { @@ -578,7 +613,6 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { rows, err = q.Execute(db, si, p.KeepConnection) if Debug { duration := time.Since(query_time) - // delta := time.Now().Sub(a) log.Printf("I! Query %d exectution time: %s", q.index, duration) } query_time = time.Now() @@ -604,6 +638,8 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { row_count := 0 for rows.Next() { + var timestamp time.Time + if err = rows.Err(); err != nil { return err } @@ -619,13 +655,13 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { // use database server as host, not the local host tags["host"] = p.Hosts[si] - err = q.ParseRow(tags, fields) - acc.AddFields(q.Measurement, fields, tags, query_time) - - // err = q.ParseRow(query_time, p.Hosts[si], acc) + timestamp, err = q.ParseRow(tags, fields, query_time) if err != nil { return err } + + acc.AddFields(q.Measurement, fields, tags, timestamp) + row_count += 1 } if Debug { From a1d618587c281a40d9113a5c30d70a8c4b1c4fa8 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 11 May 2017 16:47:53 +0200 Subject: [PATCH 06/23] added check for existence of timestamp field updated docu --- plugins/inputs/sql/README.md | 16 +++++++++++++--- plugins/inputs/sql/sql.go | 19 ++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 9c6049b068969..be2df5416159b 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -9,11 +9,13 @@ Supported drivers are go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql (MyS ## Getting started : First you need to grant read/select privileges on queried tables to the database user you use for the connection -For some not pure go drivers you may need external shared libraries and environment variables: look at sql driver implementation site +For some not pure go drivers you may need external shared libraries and environment variables: look at sql driver implementation site +For instance using oracle driver on rh linux you need to install oracle-instantclient12.2-basic-12.2.0.1.0-1.x86_64.rpm package and set ``` +export ORACLE_HOME=/usr/lib/oracle/12.2/client64 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME/lib ``` - ## Configuration: ``` @@ -50,7 +52,7 @@ For some not pure go drivers you may need external shared libraries and environm # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) - null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) + null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) sanitize = false # true: will perform some chars substitutions (false: use value as is) @@ -58,11 +60,18 @@ For some not pure go drivers you may need external shared libraries and environm ## Datatypes: +Using field_cols list the values are converted by the go database driver implementation. +In some cases this automatic conversion is not what we wxpect, therefore you can force the destination datatypes specifing the columns in the bool/int/float/time_fields lists, then if possible the plugin converts the data. +If an error in conversion occurs then telegraf exits, therefore a --test run is suggested. +## Tested Databases +Actually I run the plugin using oci8,mysql and mssql +The mechanism for get the timestamp from a table column has known problems ## Example for collect multiple counters defined as COLUMNS in a table: Here we read a table where each counter is on a different row. Each row contains a column with the name of the counter (counter_name) and a column with his value (cntr_value) and some other columns that we use as tags (instance_name,object_name) +###Config ``` [[inputs.sql]] interval = "60s" @@ -97,6 +106,7 @@ Here we read multiple counters defined on same row where the counter name is the In this example we force some counters datatypes: "MEMBERS","FIRST_CHANGE#" as integer, "BYTES" as float, "FIRST_TIME" as time. The field "UNIT" is used with the automatic driver datatype conversion. The column "ARCHIVED" is ignored +###Config ``` [[inputs.sql]] interval = "20s" diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 36ce7248f93a0..cd7b7428a67f8 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -156,7 +156,7 @@ func (s *Sql) SampleConfig() string { # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) - null_as_zero = false # true: Push null results as zeros/empty strings (false: ignore fields) + null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) sanitize = false # true: will perform some chars substitutions (false: use value as is) ` return sampleConfig @@ -170,12 +170,10 @@ func (s *Sql) Init() error { Debug = s.Debug if Debug { - log.Printf("I! Init %d servers %d queries", len(s.Servers), len(s.Query)) + log.Printf("I! Init %d servers %d queries, driver %s", len(s.Servers), len(s.Query), s.Driver) } if s.KeepConnection { s.connections = make([]*sql.DB, len(s.Servers)) - // for _, q := range s.Query { - // q.statements = make([]*sql.Stmt, len(s.Servers)) for i := 0; i < len(s.Query); i++ { s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) } @@ -230,10 +228,13 @@ func (s *Query) Init(cols []string) error { s.field_value_idx = -1 s.field_timestamp_idx = -1 + if len(s.FieldTimestamp) > 0 && !contains_str(s.FieldTimestamp, s.column_name) { + log.Printf("E! Missing given field_timestamp in columns: %s", s.FieldTimestamp) + return errors.New("Missing given field_timestamp in columns") + } if len(s.FieldName) > 0 && !contains_str(s.FieldName, s.column_name) { log.Printf("E! Missing given field_name in columns: %s", s.FieldName) - err := errors.New("Missing given field_name in columns") - return err + return errors.New("Missing given field_name in columns") } if len(s.FieldValue) > 0 && !contains_str(s.FieldValue, s.column_name) { log.Printf("E! Missing given field_value in columns: %s", s.FieldValue) @@ -306,6 +307,7 @@ func (s *Query) Init(cols []string) error { s.cells[i] = cell s.cell_refs[i] = &s.cells[i] } + if Debug { log.Printf("I! Query received %d tags and %d fields on %d columns...", s.tag_count, s.field_count, col_count) } @@ -632,7 +634,10 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { if err != nil { return err } - q.Init(cols) + err = q.Init(cols) + if err != nil { + return err + } } row_count := 0 From 42afe14baf1b0c5c15b561f9474f7b603786c1df Mon Sep 17 00:00:00 2001 From: Luca Di Stefano Date: Thu, 11 May 2017 22:22:57 +0200 Subject: [PATCH 07/23] commented oracle drivers because of the external proprietary libraries dependenciy --- plugins/inputs/sql/sql.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index cd7b7428a67f8..4a35ad9b6580b 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -16,11 +16,15 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" // database drivers here: - // _ "bitbucket.org/phiggins/db2cli" // +// _ "bitbucket.org/phiggins/db2cli" // + _ "github.com/SAP/go-hdb" + _ "github.com/mattn/go-sqlite3" +// _ "github.com/a-palchikov/sqlago" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" // pure go - _ "github.com/mattn/go-oci8" - _ "gopkg.in/rana/ora.v4" +// oracle commented because of the external proprietary libraries +// _ "github.com/mattn/go-oci8" +// _ "gopkg.in/rana/ora.v4" // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go ) From 8f78eb38f1a2a62f996108ab95c106cea5a710ca Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 11 May 2017 22:30:13 +0200 Subject: [PATCH 08/23] added go-sqlite3 driver --- plugins/inputs/sql/sql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 4a35ad9b6580b..ba3182d06df0d 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -17,7 +17,7 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" // database drivers here: // _ "bitbucket.org/phiggins/db2cli" // - _ "github.com/SAP/go-hdb" +// _ "github.com/SAP/go-hdb" _ "github.com/mattn/go-sqlite3" // _ "github.com/a-palchikov/sqlago" _ "github.com/go-sql-driver/mysql" From 88593b3c3c9552298d5301ca4ed4f5319c78cd88 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 11 May 2017 22:53:10 +0200 Subject: [PATCH 09/23] formatted code added dependencies --- Godeps | 2 ++ plugins/inputs/all/all.go | 2 +- plugins/inputs/sql/sql.go | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Godeps b/Godeps index aa9ace1ab1f3c..3be8d57dd7467 100644 --- a/Godeps +++ b/Godeps @@ -30,6 +30,8 @@ github.com/kardianos/osext c2c54e542fb797ad986b31721e1baedf214ca413 github.com/kardianos/service 6d3a0ee7d3425d9d835debc51a0ca1ffa28f4893 github.com/kballard/go-shellquote d8ec1a69a250a17bb0e419c386eac1f3711dc142 github.com/klauspost/crc32 cb6bfca970f6908083f26f39a79009d608efd5cd +github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36 +github.com/mattn/go-sqlite3 cf7286f069c3ef596efcc87781a4653a2e7607bd github.com/matttproud/golang_protobuf_extensions c12348ce28de40eed0136aa2b644d0ee0650e56c github.com/miekg/dns 99f84ae56e75126dd77e5de4fae2ea034a468ca1 github.com/naoina/go-stringutil 6b638e95a32d0c1131db0e7fe83775cbea4a0d0b diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index ffc93410ce6ca..4023eba89d7e0 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -71,7 +71,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/snmp" _ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy" _ "github.com/influxdata/telegraf/plugins/inputs/socket_listener" - _ "github.com/influxdata/telegraf/plugins/inputs/sql" + _ "github.com/influxdata/telegraf/plugins/inputs/sql" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" _ "github.com/influxdata/telegraf/plugins/inputs/statsd" _ "github.com/influxdata/telegraf/plugins/inputs/sysstat" diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index ba3182d06df0d..16a37b1eec1e8 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -16,15 +16,15 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" // database drivers here: -// _ "bitbucket.org/phiggins/db2cli" // -// _ "github.com/SAP/go-hdb" + // _ "bitbucket.org/phiggins/db2cli" // + // _ "github.com/SAP/go-hdb" _ "github.com/mattn/go-sqlite3" -// _ "github.com/a-palchikov/sqlago" + // _ "github.com/a-palchikov/sqlago" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" // pure go -// oracle commented because of the external proprietary libraries -// _ "github.com/mattn/go-oci8" -// _ "gopkg.in/rana/ora.v4" + // oracle commented because of the external proprietary libraries + // _ "github.com/mattn/go-oci8" + // _ "gopkg.in/rana/ora.v4" // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go ) From 18e62876bda7590e359621d31392860435fd41cb Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Fri, 12 May 2017 09:47:13 +0200 Subject: [PATCH 10/23] fixed string conversion when cell has []byte updated readme with warn on not imported oracle libs --- plugins/inputs/sql/README.md | 6 +- plugins/inputs/sql/sql.go | 147 ++++++++++++++++++++++------------- 2 files changed, 96 insertions(+), 57 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index be2df5416159b..2d3da9f9f3e94 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -7,14 +7,17 @@ Supported drivers are go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql (MyS ``` ## Getting started : - First you need to grant read/select privileges on queried tables to the database user you use for the connection + +### Non pure go drivers For some not pure go drivers you may need external shared libraries and environment variables: look at sql driver implementation site For instance using oracle driver on rh linux you need to install oracle-instantclient12.2-basic-12.2.0.1.0-1.x86_64.rpm package and set ``` export ORACLE_HOME=/usr/lib/oracle/12.2/client64 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME/lib ``` +Actually the dependencies to all those drivers (oracle,db2,sap) are commented in the sql.go source. You can enable it, just remove the comment and perform a 'go get ' and recompile telegraf + ## Configuration: @@ -57,6 +60,7 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME/lib ``` +sql_script is read only once, if you change the script you need to restart telegraf ## Datatypes: diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 16a37b1eec1e8..de80fd1a6d3c3 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -4,29 +4,28 @@ import ( "bytes" "database/sql" "errors" - // "github.com/gchaincl/dotsql" "fmt" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" "log" "os" "reflect" "strconv" "strings" "time" - - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/inputs" // database drivers here: - // _ "bitbucket.org/phiggins/db2cli" // - // _ "github.com/SAP/go-hdb" _ "github.com/mattn/go-sqlite3" // _ "github.com/a-palchikov/sqlago" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" // pure go - // oracle commented because of the external proprietary libraries - // _ "github.com/mattn/go-oci8" - // _ "gopkg.in/rana/ora.v4" // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go + // oracle commented because of the external proprietary libraries dependencies + // _ "github.com/mattn/go-oci8" + // _ "gopkg.in/rana/ora.v4" + // the following commented because of the external proprietary libraries dependencies + // _ "bitbucket.org/phiggins/db2cli" // + // _ "github.com/SAP/go-hdb" ) const TYPE_STRING = 1 @@ -271,8 +270,8 @@ func (s *Query) Init(cols []string) error { field_matched = false s.tag_idx[s.tag_count] = i s.tag_count++ - cell = new(sql.RawBytes) - // cell = new(string); + // cell = new(sql.RawBytes) + cell = new(string) } else if contains_str(s.column_name[i], s.IntFields) { s.field_type[s.field_count] = TYPE_INT cell = new(sql.RawBytes) @@ -319,6 +318,20 @@ func (s *Query) Init(cols []string) error { return nil } +func ConvertString(name string, cell interface{}) (string, bool) { + value, ok := cell.(string) + if !ok { + barr, ok := cell.([]byte) + value = string(barr) + if !ok { + value = fmt.Sprintf("%v", cell) + ok = true + log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + } + } + return value, ok +} + func (s *Query) ConvertField(name string, cell interface{}, field_type int, NullAsZero bool) (interface{}, error) { var value interface{} var ok bool @@ -346,16 +359,25 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null value, err = strconv.ParseBool(str) } break - // case TYPE_TIME: - // value = cell - // break - // case TYPE_STRING: - // value = cell - // break + case TYPE_TIME: + value, ok = cell.(time.Time) + // TODO convert to ns/ms/s/us?? + if !ok { + var intvalue int64 + intvalue, ok = value.(int64) + if !ok { + log.Printf("E! Unable to convert timestamp '%s' type %s", s.column_name[s.field_timestamp_idx], reflect.TypeOf(cell).Kind()) + } else { + value = time.Unix(intvalue, 0) + } + } + break + case TYPE_STRING: + value, ok = ConvertString(name, cell) + break default: value = cell } - } else if NullAsZero { switch field_type { case TYPE_AUTO: @@ -376,13 +398,10 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null } } if !ok { - cell_type := reflect.TypeOf(cell).Kind() - - log.Printf("E! converting field name '%s' type %d %s into string", name, field_type, cell_type) err = errors.New("Error converting field into string") - return nil, err } if err != nil { + log.Printf("E! converting name '%s' type %s into type %d, raw data '%s'", name, reflect.TypeOf(cell).Kind(), field_type, fmt.Sprintf("%v", cell)) return nil, err } return value, nil @@ -390,48 +409,66 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, timestamp time.Time) (time.Time, error) { if s.field_timestamp_idx >= 0 { - var ok bool // get the value of timestamp field cell := s.cells[s.field_timestamp_idx] - value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], cell, TYPE_AUTO, s.NullAsZero) + value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], cell, TYPE_TIME, false) if err != nil { - return timestamp, err + return timestamp, errors.New("Cannot convert timestamp") } - timestamp, ok = value.(time.Time) - // TODO convert to ns/ms/s/us?? - if !ok { - log.Printf("W! Unable to convert timestamp '%s' to time.Time", s.column_name[s.field_timestamp_idx]) - var intvalue int64 - intvalue, ok = value.(int64) - if !ok { - cell_type := reflect.TypeOf(cell).Kind() + timestamp, _ = value.(time.Time) - log.Printf("E! Unable to convert timestamp '%s' type %s", s.column_name[s.field_timestamp_idx], cell_type) - return timestamp, errors.New("Cannot convert timestamp") - } - timestamp = time.Unix(intvalue, 0) - } + // var ok bool + // value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], cell, TYPE_AUTO, s.NullAsZero) + // if err != nil { + // return timestamp, err + // } + // timestamp, ok = value.(time.Time) + // // TODO convert to ns/ms/s/us?? + // if !ok { + // log.Printf("W! Unable to convert timestamp '%s' to time.Time", s.column_name[s.field_timestamp_idx]) + // var intvalue int64 + // intvalue, ok = value.(int64) + // if !ok { + // cell_type := reflect.TypeOf(cell).Kind() + // log.Printf("E! Unable to convert timestamp '%s' type %s", s.column_name[s.field_timestamp_idx], cell_type) + // return timestamp, errors.New("Cannot convert timestamp") + // } + // timestamp = time.Unix(intvalue, 0) + // } } // fill tags for i := 0; i < s.tag_count; i++ { cell := s.cells[s.tag_idx[i]] if cell != nil { - // tags are always strings + // tags should be always strings name := s.column_name[s.tag_idx[i]] - //TODO set flag for force tag data conversion? + // //TODO set flag for force tag data conversion? // value, ok := cell.(string) // if !ok { - // log.Printf("W! converting tag %d '%s' type %d", s.tag_idx[i], name, TYPE_STRING) - // return nil - // // return errors.New("Cannot convert tag") + // barr, ok := cell.([]byte) + // value = string(barr) + // if !ok { + // value = fmt.Sprintf("%v", cell) + // log.Printf("W! converting tag %d '%s' type %s", s.tag_idx[i], name, reflect.TypeOf(cell).Kind()) + // // return nil // skips the row + // // return errors.New("Cannot convert tag") // break the run + // } // } - value := fmt.Sprintf("%v", cell) - if s.Sanitize { - tags[name] = sanitize(value) + + value, ok := ConvertString(name, cell) + if !ok { + log.Printf("W! ignored tag %s", name) + // ignoring tag is correct? + // return nil // skips the row + // return errors.New("Cannot convert tag") // break the run } else { - tags[name] = value + if s.Sanitize { + tags[name] = sanitize(value) + } else { + tags[name] = value + } } } } @@ -439,7 +476,8 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, if s.field_name_idx >= 0 { // get the name of the field from value on column cell := s.cells[s.field_name_idx] - name, ok := cell.(string) + name, ok := ConvertString(s.column_name[s.field_name_idx], cell) + // name, ok := cell.(string) if !ok { log.Printf("W! converting field name '%s'", s.column_name[s.field_name_idx]) return timestamp, nil @@ -560,14 +598,6 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } rows, err = db.Query(q.Query) } - // } else if len(q.QueryScript) > 0 { - // // Loads queries from file - // var dot *dotsql.DotSql - // dot, err = dotsql.LoadFromFile(q.QueryScript) - // if err != nil { - // return nil, err - // } - // rows, err = dot.Query(db, "find-users-by-email") } else { log.Printf("W! No query to execute %d", q.index) // err = errors.New("No query to execute") @@ -671,6 +701,11 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { acc.AddFields(q.Measurement, fields, tags, timestamp) + // fieldsG := map[string]interface{}{ + // "usage_user": 100 * (cts.User - lastCts.User - (cts.Guest - lastCts.Guest)) / totalDelta, + // } + // acc.AddGauge("cpu", fieldsG, tags, now) + row_count += 1 } if Debug { From 113c3e8def0bbe9b45bd7635609122d986d98021 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Fri, 12 May 2017 10:01:11 +0200 Subject: [PATCH 11/23] log entry for conversion moved to debugl level --- plugins/inputs/sql/sql.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index de80fd1a6d3c3..c8770a6a50198 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -321,12 +321,15 @@ func (s *Query) Init(cols []string) error { func ConvertString(name string, cell interface{}) (string, bool) { value, ok := cell.(string) if !ok { - barr, ok := cell.([]byte) + var barr []byte + barr, ok = cell.([]byte) value = string(barr) if !ok { value = fmt.Sprintf("%v", cell) ok = true - log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + if Debug { + log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + } } } return value, ok From c89f7bc818a1660decd271b4f2dcdebbf8cdb475 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Fri, 12 May 2017 14:45:05 +0200 Subject: [PATCH 12/23] removed comments implemented null_as_zero for all datatypes removed duplicated log for error in conversion now the field_value column can be forced to a choosen datatype --- plugins/inputs/sql/README.md | 13 ++- plugins/inputs/sql/sql.go | 164 ++++++++++++++++------------------- 2 files changed, 85 insertions(+), 92 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 2d3da9f9f3e94..29f8ed3dc74b9 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -27,7 +27,7 @@ Actually the dependencies to all those drivers (oracle,db2,sap) are commented in # debug=false # Enables very verbose output ## Database Driver - driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 | ora.v4 (Oracle), mysql, postgres # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) ## Server DSNs @@ -62,6 +62,8 @@ Actually the dependencies to all those drivers (oracle,db2,sap) are commented in ``` sql_script is read only once, if you change the script you need to restart telegraf +## Field names +Field names are the same of the relative column name or taken from value of a column. If there is the need of rename the fields, just do it in the sql, try to use an ' AS ' . ## Datatypes: Using field_cols list the values are converted by the go database driver implementation. @@ -72,7 +74,7 @@ If an error in conversion occurs then telegraf exits, therefore a --test run is Actually I run the plugin using oci8,mysql and mssql The mechanism for get the timestamp from a table column has known problems -## Example for collect multiple counters defined as COLUMNS in a table: +## Example for collect multiple counters defined as COLUMNS in a table (vertical counter structure): Here we read a table where each counter is on a different row. Each row contains a column with the name of the counter (counter_name) and a column with his value (cntr_value) and some other columns that we use as tags (instance_name,object_name) ###Config @@ -105,7 +107,7 @@ Here we read a table where each counter is on a different row. Each row contains ... ``` -## Example for collect multiple counters defined as ROWS in a table: +## Example for collect multiple counters defined as ROWS in a table (horizontal counter structure): Here we read multiple counters defined on same row where the counter name is the name of his column. In this example we force some counters datatypes: "MEMBERS","FIRST_CHANGE#" as integer, "BYTES" as float, "FIRST_TIME" as time. The field "UNIT" is used with the automatic driver datatype conversion. The column "ARCHIVED" is ignored @@ -139,3 +141,8 @@ The column "ARCHIVED" is ignored ``` +## TODO +Give the possibility to define parameters to pass to the prepared statement +Get the host tag value automatically parsing the connection DSN string +Implement tests + diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index c8770a6a50198..0ded20e5bdf89 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -62,9 +62,9 @@ type Query struct { // QueryScript string - // internal data + // -------- internal data ----------- statements []*sql.Stmt - // Parameters []string + // Parameters []string //TODO column_name []string cell_refs []interface{} @@ -79,6 +79,7 @@ type Query struct { field_name_idx int field_value_idx int + field_value_type int field_timestamp_idx int index int @@ -202,10 +203,16 @@ func (s *Query) Init(cols []string) error { if Debug { log.Printf("I! Init Query %d with %d columns", s.index, len(cols)) } - s.column_name = cols + //Define index of tags and fields and keep it for reuse + s.column_name = cols + + // init the arrays for store row data col_count := len(s.column_name) + s.cells = make([]interface{}, col_count) + s.cell_refs = make([]interface{}, col_count) + // init the arrays for store field/tag infos expected_tag_count := len(s.TagCols) var expected_field_count int if !s.IgnoreOtherFields { @@ -224,9 +231,7 @@ func (s *Query) Init(cols []string) error { s.tag_count = 0 s.field_count = 0 - s.cells = make([]interface{}, col_count) - s.cell_refs = make([]interface{}, col_count) - + // prepare vars for vertical counter parsing s.field_name_idx = -1 s.field_value_idx = -1 s.field_timestamp_idx = -1 @@ -247,23 +252,14 @@ func (s *Query) Init(cols []string) error { return errors.New("Both field_name and field_value should be set") } + // fill columns info var cell interface{} for i := 0; i < col_count; i++ { - if Debug { - log.Printf("I! Field %s %d", s.column_name[i], i) - } + dest_type := TYPE_AUTO field_matched := true - if s.column_name[i] == s.FieldName { - s.field_name_idx = i - } - if s.column_name[i] == s.FieldValue { - s.field_value_idx = i - // TODO force datatype? - } - if s.column_name[i] == s.FieldTimestamp { - s.field_timestamp_idx = i - // TODO force datatype? + if Debug { + log.Printf("I! Field %s %d", s.column_name[i], i) } if contains_str(s.column_name[i], s.TagCols) { @@ -273,27 +269,27 @@ func (s *Query) Init(cols []string) error { // cell = new(sql.RawBytes) cell = new(string) } else if contains_str(s.column_name[i], s.IntFields) { - s.field_type[s.field_count] = TYPE_INT + dest_type = TYPE_INT cell = new(sql.RawBytes) // cell = new(int); } else if contains_str(s.column_name[i], s.FloatFields) { - s.field_type[s.field_count] = TYPE_FLOAT + dest_type = TYPE_FLOAT // cell = new(float64); cell = new(sql.RawBytes) } else if contains_str(s.column_name[i], s.TimeFields) { //TODO as number? - s.field_type[s.field_count] = TYPE_TIME - cell = new(string) - // cell = new(sql.RawBytes) + dest_type = TYPE_TIME + // cell = new(string) + cell = new(sql.RawBytes) } else if contains_str(s.column_name[i], s.BoolFields) { - s.field_type[s.field_count] = TYPE_BOOL + dest_type = TYPE_BOOL // cell = new(bool); cell = new(sql.RawBytes) } else if contains_str(s.column_name[i], s.FieldCols) { - s.field_type[s.field_count] = TYPE_AUTO + dest_type = TYPE_AUTO cell = new(sql.RawBytes) } else if !s.IgnoreOtherFields { - s.field_type[s.field_count] = TYPE_AUTO + dest_type = TYPE_AUTO cell = new(sql.RawBytes) // cell = new(string); } else { @@ -303,7 +299,23 @@ func (s *Query) Init(cols []string) error { log.Printf("I! Skipped field %s", s.column_name[i]) } } + + if s.column_name[i] == s.FieldName { + s.field_name_idx = i + field_matched = false + } + if s.column_name[i] == s.FieldValue { + s.field_value_idx = i + s.field_value_type = dest_type + field_matched = false + } + if s.column_name[i] == s.FieldTimestamp { + s.field_timestamp_idx = i + field_matched = false + } + if field_matched { + s.field_type[s.field_count] = dest_type s.field_idx[s.field_count] = i s.field_count++ } @@ -368,9 +380,7 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null if !ok { var intvalue int64 intvalue, ok = value.(int64) - if !ok { - log.Printf("E! Unable to convert timestamp '%s' type %s", s.column_name[s.field_timestamp_idx], reflect.TypeOf(cell).Kind()) - } else { + if ok { value = time.Unix(intvalue, 0) } } @@ -387,6 +397,15 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null case TYPE_STRING: value = "" break + case TYPE_INT: + value = 0i + case TYPE_FLOAT: + value = 0.0 + case TYPE_BOOL: + value = false + case TYPE_TIME: + value = time.Unix(0, 0) + break default: value = 0 } @@ -419,68 +438,34 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, return timestamp, errors.New("Cannot convert timestamp") } timestamp, _ = value.(time.Time) - - // var ok bool - // value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], cell, TYPE_AUTO, s.NullAsZero) - // if err != nil { - // return timestamp, err - // } - // timestamp, ok = value.(time.Time) - // // TODO convert to ns/ms/s/us?? - // if !ok { - // log.Printf("W! Unable to convert timestamp '%s' to time.Time", s.column_name[s.field_timestamp_idx]) - // var intvalue int64 - // intvalue, ok = value.(int64) - // if !ok { - // cell_type := reflect.TypeOf(cell).Kind() - // log.Printf("E! Unable to convert timestamp '%s' type %s", s.column_name[s.field_timestamp_idx], cell_type) - // return timestamp, errors.New("Cannot convert timestamp") - // } - // timestamp = time.Unix(intvalue, 0) - // } } // fill tags for i := 0; i < s.tag_count; i++ { cell := s.cells[s.tag_idx[i]] - if cell != nil { - // tags should be always strings - name := s.column_name[s.tag_idx[i]] - - // //TODO set flag for force tag data conversion? - // value, ok := cell.(string) - // if !ok { - // barr, ok := cell.([]byte) - // value = string(barr) - // if !ok { - // value = fmt.Sprintf("%v", cell) - // log.Printf("W! converting tag %d '%s' type %s", s.tag_idx[i], name, reflect.TypeOf(cell).Kind()) - // // return nil // skips the row - // // return errors.New("Cannot convert tag") // break the run - // } - // } - - value, ok := ConvertString(name, cell) - if !ok { - log.Printf("W! ignored tag %s", name) - // ignoring tag is correct? - // return nil // skips the row - // return errors.New("Cannot convert tag") // break the run + // if cell != nil { + // tags should be always strings + name := s.column_name[s.tag_idx[i]] + value, ok := ConvertString(name, cell) + if !ok { + log.Printf("W! ignored tag %s", name) + // ignoring tag is correct? + // return nil // skips the row + // return errors.New("Cannot convert tag") // break the run + } else { + if s.Sanitize { + tags[name] = sanitize(value) } else { - if s.Sanitize { - tags[name] = sanitize(value) - } else { - tags[name] = value - } + tags[name] = value } } + // } } if s.field_name_idx >= 0 { // get the name of the field from value on column cell := s.cells[s.field_name_idx] name, ok := ConvertString(s.column_name[s.field_name_idx], cell) - // name, ok := cell.(string) if !ok { log.Printf("W! converting field name '%s'", s.column_name[s.field_name_idx]) return timestamp, nil @@ -493,23 +478,24 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, // get the value of field cell = s.cells[s.field_value_idx] - value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, TYPE_AUTO, s.NullAsZero) // TODO set forced field type + value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, s.field_value_type, s.NullAsZero) // TODO set forced field type if err != nil { return timestamp, err } fields[name] = value - } else { - // fill fields from column values - for i := 0; i < s.field_count; i++ { - cell := s.cells[s.field_idx[i]] - name := s.column_name[s.field_idx[i]] - value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) - if err != nil { - return timestamp, err - } - fields[name] = value + } + // else { + // fill fields from column values + for i := 0; i < s.field_count; i++ { + cell := s.cells[s.field_idx[i]] + name := s.column_name[s.field_idx[i]] + value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) + if err != nil { + return timestamp, err } + fields[name] = value } + // } return timestamp, nil } From 6c5f2f0b613480fe76963d64f956af131247b600 Mon Sep 17 00:00:00 2001 From: Luca Di Stefano Date: Sun, 14 May 2017 23:43:46 +0200 Subject: [PATCH 13/23] removed unused comments beautified log --- plugins/inputs/sql/sql.go | 46 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 0ded20e5bdf89..4e78289e2f208 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -182,13 +182,13 @@ func (s *Sql) Init() error { s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) } } - for i := 0; i < len(s.Servers); i++ { - //TODO get host from server - // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") - // fmt.Println(match) - // addr, err := net.LookupHost("198.252.206.16") + // for i := 0; i < len(s.Servers); i++ { + //TODO get host from server + // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") + // fmt.Println(match) + // addr, err := net.LookupHost("198.252.206.16") - } + // } if len(s.Servers) > 0 && len(s.Servers) != len(s.Hosts) { return errors.New("For each server a host should be specified") @@ -221,10 +221,6 @@ func (s *Query) Init(cols []string) error { expected_field_count = len(s.FieldCols) + len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) } - if Debug { - log.Printf("I! Extpected %d tags and %d fields", expected_tag_count, expected_field_count) - } - s.tag_idx = make([]int, expected_tag_count) s.field_idx = make([]int, expected_field_count) s.field_type = make([]int, expected_field_count) @@ -258,10 +254,6 @@ func (s *Query) Init(cols []string) error { dest_type := TYPE_AUTO field_matched := true - if Debug { - log.Printf("I! Field %s %d", s.column_name[i], i) - } - if contains_str(s.column_name[i], s.TagCols) { field_matched = false s.tag_idx[s.tag_count] = i @@ -300,6 +292,10 @@ func (s *Query) Init(cols []string) error { } } + if Debug && !field_matched { + log.Printf("I! Column %d '%s' dest type %d", i, s.column_name[i], dest_type) + } + if s.column_name[i] == s.FieldName { s.field_name_idx = i field_matched = false @@ -376,7 +372,7 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null break case TYPE_TIME: value, ok = cell.(time.Time) - // TODO convert to ns/ms/s/us?? + // TODO convert to s/ms/us/ns?? if !ok { var intvalue int64 intvalue, ok = value.(int64) @@ -411,7 +407,7 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null } if Debug { - log.Printf("I! forcing to 0 field name '%s' type %d", name, field_type) + log.Printf("I! forcing to %s field name '%s' type %d", fmt.Sprintf("%v", value), name, field_type) } } else { value = nil @@ -478,13 +474,13 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, // get the value of field cell = s.cells[s.field_value_idx] - value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, s.field_value_type, s.NullAsZero) // TODO set forced field type + value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, s.field_value_type, s.NullAsZero) if err != nil { return timestamp, err } fields[name] = value } - // else { + // fill fields from column values for i := 0; i < s.field_count; i++ { cell := s.cells[s.field_idx[i]] @@ -495,7 +491,7 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, } fields[name] = value } - // } + return timestamp, nil } @@ -544,7 +540,7 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err // read query from sql script and put it in query string if len(q.QueryScript) > 0 && len(q.Query) == 0 { if _, err := os.Stat(q.QueryScript); os.IsNotExist(err) { - log.Printf("E! SQL script not exists '%s'...", q.QueryScript) + log.Printf("E! SQL script file not exists '%s'...", q.QueryScript) return nil, err } filerc, err := os.Open(q.QueryScript) @@ -577,7 +573,7 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err // execute prepared statement if Debug { - log.Printf("I! Performing query '%s'...", q.Query) + log.Printf("I! Performing query:\n\t\t%s\n...", q.Query) } rows, err = q.statements[si].Query() } else { @@ -637,8 +633,7 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { query_time = time.Now() rows, err = q.Execute(db, si, p.KeepConnection) if Debug { - duration := time.Since(query_time) - log.Printf("I! Query %d exectution time: %s", q.index, duration) + log.Printf("I! Query %d exectution time: %s", q.index, time.Since(query_time)) } query_time = time.Now() @@ -693,13 +688,12 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { // fieldsG := map[string]interface{}{ // "usage_user": 100 * (cts.User - lastCts.User - (cts.Guest - lastCts.Guest)) / totalDelta, // } - // acc.AddGauge("cpu", fieldsG, tags, now) + // acc.AddGauge("cpu", fieldsG, tags, now) // TODO use gauge too? row_count += 1 } if Debug { - duration := time.Since(query_time) - log.Printf("I! Query %d on %s found %d rows written in %s... processing duration %s", q.index, p.Hosts[si], row_count, q.Measurement, duration) + log.Printf("I! Query %d on %s found %d rows written in %s... processing duration %s", q.index, p.Hosts[si], row_count, q.Measurement, time.Since(query_time)) } } } From cae1b578992ec47799378a374778fe5a24828761 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Mon, 15 May 2017 09:53:21 +0200 Subject: [PATCH 14/23] use golang 1.8 plugins feature for load external drivers (tested with oci8) --- plugins/inputs/sql/README.md | 46 ++++++++++++++++++++++++++++++++---- plugins/inputs/sql/sql.go | 17 +++++++++++-- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 29f8ed3dc74b9..75836c1542935 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -10,13 +10,48 @@ Supported drivers are go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql (MyS First you need to grant read/select privileges on queried tables to the database user you use for the connection ### Non pure go drivers -For some not pure go drivers you may need external shared libraries and environment variables: look at sql driver implementation site -For instance using oracle driver on rh linux you need to install oracle-instantclient12.2-basic-12.2.0.1.0-1.x86_64.rpm package and set +For some not pure go drivers you may need external shared libraries and environment variables: look at sql driver implementation site +Actually the dependencies to all those drivers (oracle,db2,sap) are commented in the sql.go source. You can enable it, just remove the comment and perform a 'go get ' and recompile telegraf. As alternative you can use the 'golang 1.8 plugins feature'like described here below + +### Oracle driver with golang 1.8 plugins feature +Follow the docu in https://github.com/mattn/go-oci8 for build the oci8 driver. +If all is going well now golang oci8 driver is compiled and linked against oracle shared libs. But not linked in telegraf. + +For let i use in telegraf, do the following: +create a file plugin.go with this content: + ``` -export ORACLE_HOME=/usr/lib/oracle/12.2/client64 -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME/lib +package main + +import "C" + +import ( + "log" + "fmt" + "github.com/mattn/go-oci8" + // .. here you can agg other proprietary driver +) + +func main() { + o := oci8.OCI8Driver{} + log.Printf("I! Loaded shared lib '%s'", fmt.Sprintf("%v", o)) +} +``` +build it with +``` +mkdir $GOPATH/lib +go build -buildmode=plugin -o $GOPATH/lib/oci8_go.so plugin.go ``` -Actually the dependencies to all those drivers (oracle,db2,sap) are commented in the sql.go source. You can enable it, just remove the comment and perform a 'go get ' and recompile telegraf +in the input plugin configuration specigy the path of the created shared lib +``` +[[inputs.sql]] + ... + driver = "oci8" + shared_lib = "/home/luca/.gocode/lib/oci8_go.so" + ... +``` + +The steps of above can be reused in a similar way for other proprietary and non proprietary drivers ## Configuration: @@ -28,6 +63,7 @@ Actually the dependencies to all those drivers (oracle,db2,sap) are commented in ## Database Driver driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 | ora.v4 (Oracle), mysql, postgres + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # path to the golang 1.8 plugin shared lib # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) ## Server DSNs diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 4e78289e2f208..d7ae1f6f3a352 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -20,8 +20,9 @@ import ( _ "github.com/lib/pq" // pure go // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go + "plugin" // oracle commented because of the external proprietary libraries dependencies - // _ "github.com/mattn/go-oci8" + // _ "github.com/mattn/go-oci8" // TODO use golang 1.8 plugins for load dyn the shared lib? // _ "gopkg.in/rana/ora.v4" // the following commented because of the external proprietary libraries dependencies // _ "bitbucket.org/phiggins/db2cli" // @@ -61,10 +62,10 @@ type Query struct { Sanitize bool // QueryScript string + // Parameters []string //TODO // -------- internal data ----------- statements []*sql.Stmt - // Parameters []string //TODO column_name []string cell_refs []interface{} @@ -89,6 +90,7 @@ type Sql struct { Hosts []string Driver string + SharedLib string Servers []string KeepConnection bool @@ -176,6 +178,17 @@ func (s *Sql) Init() error { if Debug { log.Printf("I! Init %d servers %d queries, driver %s", len(s.Servers), len(s.Query), s.Driver) } + + if len(s.SharedLib) > 0 { + _, err := plugin.Open(s.SharedLib) + if err != nil { + panic(err) + } + if Debug { + log.Printf("I! Loaded shared lib %s '%s'", s.SharedLib) + } + } + if s.KeepConnection { s.connections = make([]*sql.DB, len(s.Servers)) for i := 0; i < len(s.Query); i++ { From 135ea88ef16941ccae80909f79b47e1437ebc0dc Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Mon, 15 May 2017 11:09:50 +0200 Subject: [PATCH 15/23] fixed printf error --- plugins/inputs/sql/sql.go | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index d7ae1f6f3a352..57f91a3c3716c 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -8,7 +8,9 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "log" + // "net/url" "os" + "plugin" "reflect" "strconv" "strings" @@ -20,7 +22,6 @@ import ( _ "github.com/lib/pq" // pure go // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go - "plugin" // oracle commented because of the external proprietary libraries dependencies // _ "github.com/mattn/go-oci8" // TODO use golang 1.8 plugins for load dyn the shared lib? // _ "gopkg.in/rana/ora.v4" @@ -172,6 +173,30 @@ func (_ *Sql) Description() string { return "SQL Plugin" } +type DSN struct { + host string + dbname string +} + +//func ParseDSN(dsn string) (*DSN, error) { +// +// url, err := url.Parse(dsn) +// if err != nil { +// return nil, err +// } +// pdsn := &DSN{} +// pdsn.host = url.Host +// pdsn.dbname = url.Path +// return pdsn, err +// +// // prm := &p.SessionPrm{Host: url.Host} +// // +// // if url.User != nil { +// // pdsn.Username = url.User.Username() +// // prm.Password, _ = url.User.Password() +// // } +//} + func (s *Sql) Init() error { Debug = s.Debug @@ -185,7 +210,7 @@ func (s *Sql) Init() error { panic(err) } if Debug { - log.Printf("I! Loaded shared lib %s '%s'", s.SharedLib) + log.Printf("I! Loaded shared lib '%s'", s.SharedLib) } } @@ -195,13 +220,23 @@ func (s *Sql) Init() error { s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) } } - // for i := 0; i < len(s.Servers); i++ { - //TODO get host from server - // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") - // fmt.Println(match) - // addr, err := net.LookupHost("198.252.206.16") + for i := 0; i < len(s.Servers); i++ { + // c, err := ParseDSN(s.Servers[i]) + // if err == nil { + // log.Printf("Host %s Database %s", c.host, c.dbname) + // } else { + // panic(err) + // } - // } + //TODO get host from server + // mysql servers = ["nprobe:nprobe@tcp(neteye.wp.lan:3307)/nprobe"] + // "postgres://nprobe:nprobe@rue-test/nprobe?sslmode=disable" + // oracle telegraf/monitor@10.62.6.1:1522/tunapit + // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") + // fmt.Println(match) + // addr, err := net.LookupHost("198.252.206.16") + + } if len(s.Servers) > 0 && len(s.Servers) != len(s.Hosts) { return errors.New("For each server a host should be specified") From 9bd605aaaeedfc930459bdd312ec37efccd629cd Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Mon, 15 May 2017 17:24:20 +0200 Subject: [PATCH 16/23] avoid to overwrite host tag if already filled add dbname tag better log/handle nil string --- plugins/inputs/sql/README.md | 9 +-- plugins/inputs/sql/sql.go | 153 ++++++++++++++++++++++++----------- 2 files changed, 107 insertions(+), 55 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 75836c1542935..1b34c2682bc0b 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -27,14 +27,11 @@ import "C" import ( "log" - "fmt" - "github.com/mattn/go-oci8" - // .. here you can agg other proprietary driver + _ "github.com/mattn/go-oci8" + // .. here you can add import to other drivers ) - func main() { - o := oci8.OCI8Driver{} - log.Printf("I! Loaded shared lib '%s'", fmt.Sprintf("%v", o)) + log.Printf("I! Loaded plugin of shared libs") } ``` build it with diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 57f91a3c3716c..8eb9720b21fed 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -16,18 +16,17 @@ import ( "strings" "time" // database drivers here: - _ "github.com/mattn/go-sqlite3" - // _ "github.com/a-palchikov/sqlago" _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" // pure go + _ "github.com/mattn/go-sqlite3" // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go - // oracle commented because of the external proprietary libraries dependencies - // _ "github.com/mattn/go-oci8" // TODO use golang 1.8 plugins for load dyn the shared lib? - // _ "gopkg.in/rana/ora.v4" // the following commented because of the external proprietary libraries dependencies + // _ "github.com/mattn/go-oci8" + // _ "gopkg.in/rana/ora.v4" // _ "bitbucket.org/phiggins/db2cli" // // _ "github.com/SAP/go-hdb" + // _ "github.com/a-palchikov/sqlago" ) const TYPE_STRING = 1 @@ -88,13 +87,15 @@ type Query struct { } type Sql struct { - Hosts []string + Driver string + SharedLib string - Driver string - SharedLib string - Servers []string KeepConnection bool + Servers []string + Hosts []string + DbNames []string + Query []Query // internal @@ -136,11 +137,13 @@ func (s *Sql) SampleConfig() string { ## Database Driver driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) ## Server DSNs servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection DSN to pass to the DB driver - hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag + # hosts=["oraserver1", "oraserver2"] # optional: for each server a relative host entry should be specified and will be added as host tag + # db_names=["oraserver1", "oraserver2"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag ## Queries to perform (block below can be repeated) [[inputs.sql.query]] @@ -178,6 +181,7 @@ type DSN struct { dbname string } +// //func ParseDSN(dsn string) (*DSN, error) { // // url, err := url.Parse(dsn) @@ -189,6 +193,24 @@ type DSN struct { // pdsn.dbname = url.Path // return pdsn, err // +// res = map[string]string{} +// parts := strings.Split(dsn, ";") +// for _, part := range parts { +// if len(part) == 0 { +// continue +// } +// lst := strings.SplitN(part, "=", 2) +// name := strings.TrimSpace(strings.ToLower(lst[0])) +// if len(name) == 0 { +// continue +// } +// var value string = "" +// if len(lst) > 1 { +// value = strings.TrimSpace(lst[1]) +// } +// res[name] = value +// } +// return res // // prm := &p.SessionPrm{Host: url.Host} // // // // if url.User != nil { @@ -220,26 +242,28 @@ func (s *Sql) Init() error { s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) } } - for i := 0; i < len(s.Servers); i++ { - // c, err := ParseDSN(s.Servers[i]) - // if err == nil { - // log.Printf("Host %s Database %s", c.host, c.dbname) - // } else { - // panic(err) - // } - - //TODO get host from server - // mysql servers = ["nprobe:nprobe@tcp(neteye.wp.lan:3307)/nprobe"] - // "postgres://nprobe:nprobe@rue-test/nprobe?sslmode=disable" - // oracle telegraf/monitor@10.62.6.1:1522/tunapit - // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") - // fmt.Println(match) - // addr, err := net.LookupHost("198.252.206.16") - - } - if len(s.Servers) > 0 && len(s.Servers) != len(s.Hosts) { + // for i := 0; i < len(s.Servers); i++ { + // c, err := ParseDSN(s.Servers[i]) + // if err == nil { + // log.Printf("Host %s Database %s", c.host, c.dbname) + // } else { + // panic(err) + // } + // + // //TODO get host from server + // // mysql servers = ["nprobe:nprobe@tcp(neteye.wp.lan:3307)/nprobe"] + // // "postgres://nprobe:nprobe@rue-test/nprobe?sslmode=disable" + // // oracle telegraf/monitor@10.62.6.1:1522/tunapit + // // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") + // // fmt.Println(match) + // // addr, err := net.LookupHost("198.252.206.16") + // + // } + if len(s.Servers) > 0 && len(s.Hosts) > 0 && len(s.Hosts) != len(s.Servers) { return errors.New("For each server a host should be specified") - + } + if len(s.Servers) > 0 && len(s.DbNames) > 0 && len(s.DbNames) != len(s.Servers) { + return errors.New("For each server a db name should be specified") } return nil } @@ -375,17 +399,28 @@ func (s *Query) Init(cols []string) error { } func ConvertString(name string, cell interface{}) (string, bool) { + if cell == nil { + return "", false + } + value, ok := cell.(string) if !ok { var barr []byte barr, ok = cell.([]byte) - value = string(barr) if !ok { - value = fmt.Sprintf("%v", cell) - ok = true - if Debug { - log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + var ivalue int64 + ivalue, ok = cell.(int64) + if !ok { + value = fmt.Sprintf("%v", cell) + ok = true + if Debug { + log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + } + } else { + value = string(ivalue) } + } else { + value = string(barr) } } return value, ok @@ -487,23 +522,27 @@ func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, // fill tags for i := 0; i < s.tag_count; i++ { cell := s.cells[s.tag_idx[i]] - // if cell != nil { - // tags should be always strings name := s.column_name[s.tag_idx[i]] - value, ok := ConvertString(name, cell) - if !ok { - log.Printf("W! ignored tag %s", name) - // ignoring tag is correct? - // return nil // skips the row - // return errors.New("Cannot convert tag") // break the run - } else { - if s.Sanitize { - tags[name] = sanitize(value) + if cell != nil { + // tags should be always strings + value, ok := ConvertString(name, cell) + if ok { + if s.Sanitize { + tags[name] = sanitize(value) + } else { + tags[name] = value + } } else { - tags[name] = value + log.Printf("W! ignored tag %s", name) + // ignoring tag is correct? + // return nil // skips the row + // return errors.New("Cannot convert tag") // break the run + } + } else { + if s.NullAsZero { + tags[name] = "" } } - // } } if s.field_name_idx >= 0 { @@ -644,6 +683,8 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err func (p *Sql) Gather(acc telegraf.Accumulator) error { var err error + start_time := time.Now() + if !p.initialized { err = p.Init() if err != nil { @@ -719,12 +760,25 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { if err != nil { return err } + // collect tags and fields tags := map[string]string{} fields := map[string]interface{}{} // use database server as host, not the local host - tags["host"] = p.Hosts[si] + if len(p.Hosts) > 0 { + _, ok := tags["host"] + if !ok { + tags["host"] = p.Hosts[si] + } + } + // add dbname tag + if len(p.DbNames) > 0 { + _, ok := tags["dbname"] + if !ok { + tags["dbname"] = p.DbNames[si] + } + } timestamp, err = q.ParseRow(tags, fields, query_time) if err != nil { @@ -746,8 +800,9 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { } } if Debug { - log.Printf("I! Poll done") + log.Printf("I! Poll done, duration %s", time.Since(start_time)) } + return nil } From 7170261cf16a352de7e3eb22750861b2edcfa4ae Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Tue, 16 May 2017 14:59:51 +0200 Subject: [PATCH 17/23] added regexp matching for columns better error handling and logging added option for get measurement name from row fixed bug in int64 to str added option for ignore errors on row parsing --- plugins/inputs/sql/README.md | 53 ++++---- plugins/inputs/sql/sql.go | 255 +++++++++++++++++++++-------------- 2 files changed, 179 insertions(+), 129 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 1b34c2682bc0b..67ab15ce91520 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -2,9 +2,8 @@ The plugin executes simple queries or query scripts on multiple servers. It permits to select the tags and the fields to export, if is needed fields can be forced to a choosen datatype. -Supported drivers are go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql (MySQL), pq (Postgres) -``` -``` +Supported/integrated drivers are mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite) +Activable drivers (read below) are all golang SQL compliant drivers (see https://github.com/golang/go/wiki/SQLDrivers): for instance oci8 for Oracle ## Getting started : First you need to grant read/select privileges on queried tables to the database user you use for the connection @@ -53,45 +52,45 @@ The steps of above can be reused in a similar way for other proprietary and non ## Configuration: -``` - +``` [[inputs.sql]] # debug=false # Enables very verbose output - + ## Database Driver - driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 | ora.v4 (Oracle), mysql, postgres - # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # path to the golang 1.8 plugin shared lib + driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - + ## Server DSNs - servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection DSN to pass to the DB driver - hosts=["oraserver1", "oraserver2"] # for each server a relative host entry should be specified and will be added as host tag - + servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver + hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag + db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag + ## Queries to perform (block below can be repeated) [[inputs.sql.query]] # query has precedence on query_script, if both query and query_script are defined only query is executed - query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file # - measurement="log" # destination measurement - tag_cols=["GROUP#","NAME"] # colums used as tags - field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + measurement="connection_errors" # destination measurement + tag_cols=["application"] # colums used as tags + field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion # - # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=["FIRST_TIME"] # adds fields and forces his value as time + # time_fields=[".*_TIME"] # adds fields and forces his value as time # + # field_measurement = "CLASS" # the golumn that contains the name of the measurement # field_name = "counter_name" # the column that contains the name of the counter # field_value = "counter_value" # the column that contains the value of the counter # # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) - + # ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) sanitize = false # true: will perform some chars substitutions (false: use value as is) - - + ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) ``` sql_script is read only once, if you change the script you need to restart telegraf @@ -100,12 +99,13 @@ Field names are the same of the relative column name or taken from value of a co ## Datatypes: Using field_cols list the values are converted by the go database driver implementation. -In some cases this automatic conversion is not what we wxpect, therefore you can force the destination datatypes specifing the columns in the bool/int/float/time_fields lists, then if possible the plugin converts the data. +In some cases this automatic conversion is not what we expect, therefore you can force the destination datatypes specifing the columns in the bool/int/float/time_fields lists, then if possible the plugin converts the data. +All field lists can contain an regex for column name matching. If an error in conversion occurs then telegraf exits, therefore a --test run is suggested. ## Tested Databases -Actually I run the plugin using oci8,mysql and mssql -The mechanism for get the timestamp from a table column has known problems +Actually I run the plugin using oci8,mysql,mssql,postgres,sqlite3 + ## Example for collect multiple counters defined as COLUMNS in a table (vertical counter structure): Here we read a table where each counter is on a different row. Each row contains a column with the name of the counter (counter_name) and a column with his value (cntr_value) and some other columns that we use as tags (instance_name,object_name) @@ -179,3 +179,6 @@ Give the possibility to define parameters to pass to the prepared statement Get the host tag value automatically parsing the connection DSN string Implement tests +## ENJOY +Luca + diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 8eb9720b21fed..0fa387c88c8ad 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -8,6 +8,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "log" + "regexp" // "net/url" "os" "plugin" @@ -54,12 +55,14 @@ type Query struct { BoolFields []string TimeFields []string // - FieldName string - FieldValue string + FieldName string + FieldValue string + FieldMeasurement string // NullAsZero bool IgnoreOtherFields bool Sanitize bool + IgnoreRowErrors bool // QueryScript string // Parameters []string //TODO @@ -78,10 +81,11 @@ type Query struct { tag_count int tag_idx []int //Column indexes of tags (strings) - field_name_idx int - field_value_idx int - field_value_type int - field_timestamp_idx int + field_measurement_idx int + field_name_idx int + field_value_idx int + field_value_type int + field_timestamp_idx int index int } @@ -121,9 +125,13 @@ func sanitize(text string) string { return text } -func contains_str(key string, str_array []string) bool { - for _, b := range str_array { - if b == key { +func match_str(key string, str_array []string) bool { + for _, pattern := range str_array { + if pattern == key { + return true + } + matched, _ := regexp.MatchString(pattern, key) + if matched { return true } } @@ -136,38 +144,40 @@ func (s *Sql) SampleConfig() string { # debug=false # Enables very verbose output ## Database Driver - driver = "oci8" # required. Valid options: go-mssqldb (sqlserver) , oci8 ora.v4 (Oracle), mysql, pq (Postgres) + driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) ## Server DSNs - servers = ["telegraf/monitor@10.0.0.5:1521/thesid", "telegraf/monitor@orahost:1521/anothersid"] # required. Connection DSN to pass to the DB driver - # hosts=["oraserver1", "oraserver2"] # optional: for each server a relative host entry should be specified and will be added as host tag - # db_names=["oraserver1", "oraserver2"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag + servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver + hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag + db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag ## Queries to perform (block below can be repeated) [[inputs.sql.query]] # query has precedence on query_script, if both query and query_script are defined only query is executed - query="select GROUP#,MEMBERS,STATUS,FIRST_TIME,FIRST_CHANGE#,BYTES,ARCHIVED from v$log" + query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file # - measurement="log" # destination measurement - tag_cols=["GROUP#","NAME"] # colums used as tags - field_cols=["UNIT"] # select fields and use the database driver automatic datatype conversion + measurement="connection_errors" # destination measurement + tag_cols=["application"] # colums used as tags + field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion # # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS","BYTES"] # adds fields and forces his value as integer + # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=["FIRST_TIME"] # adds fields and forces his value as time + # time_fields=[".*_TIME"] # adds fields and forces his value as time # + # field_measurement = "CLASS" # the golumn that contains the name of the measurement # field_name = "counter_name" # the column that contains the name of the counter # field_value = "counter_value" # the column that contains the value of the counter # # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) - + # ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) sanitize = false # true: will perform some chars substitutions (false: use value as is) + ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) ` return sampleConfig } @@ -260,10 +270,10 @@ func (s *Sql) Init() error { // // } if len(s.Servers) > 0 && len(s.Hosts) > 0 && len(s.Hosts) != len(s.Servers) { - return errors.New("For each server a host should be specified") + return fmt.Errorf("For each server a host should be specified (%d/%d)", len(s.Hosts), len(s.Servers)) } if len(s.Servers) > 0 && len(s.DbNames) > 0 && len(s.DbNames) != len(s.Servers) { - return errors.New("For each server a db name should be specified") + return fmt.Errorf("For each server a db name should be specified (%d/%d)", len(s.DbNames), len(s.Servers)) } return nil } @@ -285,13 +295,16 @@ func (s *Query) Init(cols []string) error { s.cell_refs = make([]interface{}, col_count) // init the arrays for store field/tag infos - expected_tag_count := len(s.TagCols) - var expected_field_count int - if !s.IgnoreOtherFields { - expected_field_count = col_count // - expected_tag_count - } else { - expected_field_count = len(s.FieldCols) + len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) - } + // expected_tag_count := len(s.TagCols) + // var expected_field_count int + // if !s.IgnoreOtherFields { + // expected_field_count = col_count // - expected_tag_count + // } else { + // expected_field_count = len(s.FieldCols) + len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) + // } + // because of regex, now we must assume the max cols + expected_field_count := col_count + expected_tag_count := col_count s.tag_idx = make([]int, expected_tag_count) s.field_idx = make([]int, expected_field_count) @@ -303,21 +316,22 @@ func (s *Query) Init(cols []string) error { s.field_name_idx = -1 s.field_value_idx = -1 s.field_timestamp_idx = -1 + s.field_measurement_idx = -1 - if len(s.FieldTimestamp) > 0 && !contains_str(s.FieldTimestamp, s.column_name) { - log.Printf("E! Missing given field_timestamp in columns: %s", s.FieldTimestamp) - return errors.New("Missing given field_timestamp in columns") + if len(s.FieldMeasurement) > 0 && !match_str(s.FieldMeasurement, s.column_name) { + return fmt.Errorf("Missing column %s for given field_measurement", s.FieldMeasurement) + } + if len(s.FieldTimestamp) > 0 && !match_str(s.FieldTimestamp, s.column_name) { + return fmt.Errorf("Missing column %s for given field_measurement", s.FieldTimestamp) } - if len(s.FieldName) > 0 && !contains_str(s.FieldName, s.column_name) { - log.Printf("E! Missing given field_name in columns: %s", s.FieldName) - return errors.New("Missing given field_name in columns") + if len(s.FieldName) > 0 && !match_str(s.FieldName, s.column_name) { + return fmt.Errorf("Missing column %s for given field_measurement", s.FieldName) } - if len(s.FieldValue) > 0 && !contains_str(s.FieldValue, s.column_name) { - log.Printf("E! Missing given field_value in columns: %s", s.FieldValue) - return errors.New("Missing given field_value in columns") + if len(s.FieldValue) > 0 && !match_str(s.FieldValue, s.column_name) { + return fmt.Errorf("Missing column %s for given field_measurement", s.FieldValue) } if (len(s.FieldValue) > 0 && len(s.FieldName) == 0) || (len(s.FieldName) > 0 && len(s.FieldValue) == 0) { - return errors.New("Both field_name and field_value should be set") + return fmt.Errorf("Both field_name and field_value should be set") } // fill columns info @@ -326,30 +340,30 @@ func (s *Query) Init(cols []string) error { dest_type := TYPE_AUTO field_matched := true - if contains_str(s.column_name[i], s.TagCols) { + if match_str(s.column_name[i], s.TagCols) { field_matched = false s.tag_idx[s.tag_count] = i s.tag_count++ // cell = new(sql.RawBytes) cell = new(string) - } else if contains_str(s.column_name[i], s.IntFields) { + } else if match_str(s.column_name[i], s.IntFields) { dest_type = TYPE_INT cell = new(sql.RawBytes) // cell = new(int); - } else if contains_str(s.column_name[i], s.FloatFields) { + } else if match_str(s.column_name[i], s.FloatFields) { dest_type = TYPE_FLOAT // cell = new(float64); cell = new(sql.RawBytes) - } else if contains_str(s.column_name[i], s.TimeFields) { + } else if match_str(s.column_name[i], s.TimeFields) { //TODO as number? dest_type = TYPE_TIME // cell = new(string) cell = new(sql.RawBytes) - } else if contains_str(s.column_name[i], s.BoolFields) { + } else if match_str(s.column_name[i], s.BoolFields) { dest_type = TYPE_BOOL // cell = new(bool); cell = new(sql.RawBytes) - } else if contains_str(s.column_name[i], s.FieldCols) { + } else if match_str(s.column_name[i], s.FieldCols) { dest_type = TYPE_AUTO cell = new(sql.RawBytes) } else if !s.IgnoreOtherFields { @@ -368,6 +382,10 @@ func (s *Query) Init(cols []string) error { log.Printf("I! Column %d '%s' dest type %d", i, s.column_name[i], dest_type) } + if s.column_name[i] == s.FieldMeasurement { + s.field_measurement_idx = i + field_matched = false + } if s.column_name[i] == s.FieldName { s.field_name_idx = i field_matched = false @@ -392,17 +410,13 @@ func (s *Query) Init(cols []string) error { } if Debug { - log.Printf("I! Query received %d tags and %d fields on %d columns...", s.tag_count, s.field_count, col_count) + log.Printf("I! Query structure with %d tags and %d fields on %d columns...", s.tag_count, s.field_count, col_count) } return nil } func ConvertString(name string, cell interface{}) (string, bool) { - if cell == nil { - return "", false - } - value, ok := cell.(string) if !ok { var barr []byte @@ -417,7 +431,7 @@ func ConvertString(name string, cell interface{}) (string, bool) { log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) } } else { - value = string(ivalue) + value = strconv.FormatInt(ivalue, 10) } } else { value = string(barr) @@ -490,16 +504,16 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null } if Debug { - log.Printf("I! forcing to %s field name '%s' type %d", fmt.Sprintf("%v", value), name, field_type) + log.Printf("I! forcing nil value of field '%s' type %d to %s", name, field_type, fmt.Sprintf("%v", value)) } } else { value = nil if Debug { - log.Printf("I! nil value for field name '%s' type %d", name, field_type) + // log.Printf("I! nil value for field name '%s' type %d", name, field_type) } } if !ok { - err = errors.New("Error converting field into string") + err = fmt.Errorf("Error by converting field %s", name) } if err != nil { log.Printf("E! converting name '%s' type %s into type %d, raw data '%s'", name, reflect.TypeOf(cell).Kind(), field_type, fmt.Sprintf("%v", cell)) @@ -508,78 +522,97 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null return value, nil } -func (s *Query) ParseRow(tags map[string]string, fields map[string]interface{}, timestamp time.Time) (time.Time, error) { +func (s *Query) GetStringFieldValue(index int) (string, error) { + cell := s.cells[index] + if cell == nil { + if s.NullAsZero { + return "", nil + } else { + return "", fmt.Errorf("Error converting name '%s' is nil", s.column_name[index]) + } + } + + value, ok := ConvertString(s.column_name[index], cell) + if !ok { + return "", fmt.Errorf("Error converting name '%s' type %s, raw data '%s'", s.column_name[index], reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + } + + if s.Sanitize { + value = sanitize(value) + } + return value, nil +} + +func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[string]string, fields map[string]interface{}) (time.Time, string, error) { if s.field_timestamp_idx >= 0 { // get the value of timestamp field - cell := s.cells[s.field_timestamp_idx] - value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], cell, TYPE_TIME, false) + value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], s.cells[s.field_timestamp_idx], TYPE_TIME, false) if err != nil { - return timestamp, errors.New("Cannot convert timestamp") + return timestamp, measurement, errors.New("Cannot convert timestamp") } timestamp, _ = value.(time.Time) } - + // get measurement from row + if s.field_measurement_idx >= 0 { + var err error + measurement, err = s.GetStringFieldValue(s.field_measurement_idx) + if err != nil { + log.Printf("E! converting field measurement '%s'", s.column_name[s.field_measurement_idx]) + //cannot put data in correct measurement, skip line + return timestamp, measurement, err + } + } // fill tags for i := 0; i < s.tag_count; i++ { - cell := s.cells[s.tag_idx[i]] - name := s.column_name[s.tag_idx[i]] - if cell != nil { - // tags should be always strings - value, ok := ConvertString(name, cell) - if ok { - if s.Sanitize { - tags[name] = sanitize(value) - } else { - tags[name] = value - } - } else { - log.Printf("W! ignored tag %s", name) - // ignoring tag is correct? - // return nil // skips the row - // return errors.New("Cannot convert tag") // break the run - } + index := s.tag_idx[i] + name := s.column_name[index] + value, err := s.GetStringFieldValue(index) + if err != nil { + log.Printf("E! ignored tag %s", name) + // cannot put data in correct series, skip line + return timestamp, measurement, err } else { - if s.NullAsZero { - tags[name] = "" - } + // log.Printf("******! tag %s=%s %s %s", name, value, , reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) + tags[name] = value } } - + // vertical counters if s.field_name_idx >= 0 { - // get the name of the field from value on column - cell := s.cells[s.field_name_idx] - name, ok := ConvertString(s.column_name[s.field_name_idx], cell) - if !ok { - log.Printf("W! converting field name '%s'", s.column_name[s.field_name_idx]) - return timestamp, nil - // return errors.New("Cannot convert tag") - } - - if s.Sanitize { - name = sanitize(name) + // get the name of the field + name, err := s.GetStringFieldValue(s.field_name_idx) + if err != nil { + log.Printf("E! converting field name '%s'", s.column_name[s.field_name_idx]) + // cannot get name of field, skip line + return timestamp, measurement, err } // get the value of field - cell = s.cells[s.field_value_idx] - value, err := s.ConvertField(s.column_name[s.field_value_idx], cell, s.field_value_type, s.NullAsZero) + var value interface{} + value, err = s.ConvertField(s.column_name[s.field_value_idx], s.cells[s.field_value_idx], s.field_value_type, s.NullAsZero) if err != nil { - return timestamp, err + // cannot get value of column with expected datatype, skip line + return timestamp, measurement, err } + + // fill the field fields[name] = value } - + // horizontal counters // fill fields from column values for i := 0; i < s.field_count; i++ { - cell := s.cells[s.field_idx[i]] name := s.column_name[s.field_idx[i]] - value, err := s.ConvertField(name, cell, s.field_type[i], s.NullAsZero) + // get the value of field + value, err := s.ConvertField(name, s.cells[s.field_idx[i]], s.field_type[i], s.NullAsZero) if err != nil { - return timestamp, err + // cannot get value of column with expected datatype, warning and continue + log.Printf("W! converting value of field '%s'", name) + // return timestamp, measurement, err + } else { + fields[name] = value } - fields[name] = value } - return timestamp, nil + return timestamp, measurement, nil } func (p *Sql) Connect(si int) (*sql.DB, error) { @@ -761,9 +794,19 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { return err } + // for debug purposes... + // if row_count == 0 && Debug { + // for ci := 0; ci < len(q.cells); ci++ { + // if q.cells[ci] != nil { + // log.Printf("I! Column '%s' type %s, raw data '%s'", q.column_name[ci], reflect.TypeOf(q.cells[ci]).Kind(), fmt.Sprintf("%v", q.cells[ci])) + // } + // } + // } + // collect tags and fields tags := map[string]string{} fields := map[string]interface{}{} + var measurement string // use database server as host, not the local host if len(p.Hosts) > 0 { @@ -780,12 +823,16 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { } } - timestamp, err = q.ParseRow(tags, fields, query_time) + timestamp, measurement, err = q.ParseRow(query_time, q.Measurement, tags, fields) if err != nil { - return err + if q.IgnoreRowErrors { + log.Printf("W! Ignored error on row %d: %s", row_count, err) + } else { + return err + } } - acc.AddFields(q.Measurement, fields, tags, timestamp) + acc.AddFields(measurement, fields, tags, timestamp) // fieldsG := map[string]interface{}{ // "usage_user": 100 * (cts.User - lastCts.User - (cts.Guest - lastCts.Guest)) / totalDelta, @@ -795,7 +842,7 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { row_count += 1 } if Debug { - log.Printf("I! Query %d on %s found %d rows written in %s... processing duration %s", q.index, p.Hosts[si], row_count, q.Measurement, time.Since(query_time)) + log.Printf("I! Query %d on %s found %d rows written, processing duration %s", q.index, p.Hosts[si], row_count, time.Since(query_time)) } } } From 8412a755439abd166ecf405b0bd46ac045b8eeb6 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 18 May 2017 08:46:11 +0200 Subject: [PATCH 18/23] added option for get database name and host from query result --- plugins/inputs/sql/README.md | 12 +++++-- plugins/inputs/sql/sql.go | 62 ++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 67ab15ce91520..a527a537276f7 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -82,6 +82,8 @@ The steps of above can be reused in a similar way for other proprietary and non # time_fields=[".*_TIME"] # adds fields and forces his value as time # # field_measurement = "CLASS" # the golumn that contains the name of the measurement + # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value + # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value # field_name = "counter_name" # the column that contains the name of the counter # field_value = "counter_value" # the column that contains the value of the counter # @@ -175,9 +177,13 @@ The column "ARCHIVED" is ignored ``` ## TODO -Give the possibility to define parameters to pass to the prepared statement -Get the host tag value automatically parsing the connection DSN string -Implement tests +1) Implement tests +2) Keep trace of timestamp of last poll for use in the where statement +3) Group by serie if timestamp and measurement are the same within a query for perform single insert in db instead of multiple +4) Give the possibility to define parameters to pass to the prepared statement +5) Get the host and database tag value automatically parsing the connection DSN string +6) Add option for parse tags once and reuse it for all rows in a query +X) Add your needs here ..... ## ENJOY Luca diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 0fa387c88c8ad..3944d46ac2164 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -55,8 +55,10 @@ type Query struct { BoolFields []string TimeFields []string // + FieldHost string FieldName string FieldValue string + FieldDatabase string FieldMeasurement string // NullAsZero bool @@ -65,7 +67,7 @@ type Query struct { IgnoreRowErrors bool // QueryScript string - // Parameters []string //TODO + Parameters []string //TODO // -------- internal data ----------- statements []*sql.Stmt @@ -81,6 +83,8 @@ type Query struct { tag_count int tag_idx []int //Column indexes of tags (strings) + field_host_idx int + field_database_idx int field_measurement_idx int field_name_idx int field_value_idx int @@ -97,6 +101,7 @@ type Sql struct { KeepConnection bool Servers []string + Hosts []string DbNames []string @@ -150,8 +155,8 @@ func (s *Sql) SampleConfig() string { ## Server DSNs servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver - hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag - db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag + #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag + #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag ## Queries to perform (block below can be repeated) [[inputs.sql.query]] @@ -168,7 +173,9 @@ func (s *Sql) SampleConfig() string { # float_fields=["TEMPERATURE"] # adds fields and forces his value as float # time_fields=[".*_TIME"] # adds fields and forces his value as time # - # field_measurement = "CLASS" # the golumn that contains the name of the measurement + # field_measurement = "CLASS" # the column that contains the name of the measurement + # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value + # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value # field_name = "counter_name" # the column that contains the name of the counter # field_value = "counter_value" # the column that contains the value of the counter # @@ -317,7 +324,15 @@ func (s *Query) Init(cols []string) error { s.field_value_idx = -1 s.field_timestamp_idx = -1 s.field_measurement_idx = -1 + s.field_database_idx = -1 + s.field_host_idx = -1 + if len(s.FieldHost) > 0 && !match_str(s.FieldHost, s.column_name) { + return fmt.Errorf("Missing column %s for given field_host", s.FieldHost) + } + if len(s.FieldDatabase) > 0 && !match_str(s.FieldDatabase, s.column_name) { + return fmt.Errorf("Missing column %s for given field_database", s.FieldDatabase) + } if len(s.FieldMeasurement) > 0 && !match_str(s.FieldMeasurement, s.column_name) { return fmt.Errorf("Missing column %s for given field_measurement", s.FieldMeasurement) } @@ -355,7 +370,6 @@ func (s *Query) Init(cols []string) error { // cell = new(float64); cell = new(sql.RawBytes) } else if match_str(s.column_name[i], s.TimeFields) { - //TODO as number? dest_type = TYPE_TIME // cell = new(string) cell = new(sql.RawBytes) @@ -382,6 +396,14 @@ func (s *Query) Init(cols []string) error { log.Printf("I! Column %d '%s' dest type %d", i, s.column_name[i], dest_type) } + if s.column_name[i] == s.FieldHost { + s.field_host_idx = i + field_matched = false + } + if s.column_name[i] == s.FieldDatabase { + s.field_database_idx = i + field_matched = false + } if s.column_name[i] == s.FieldMeasurement { s.field_measurement_idx = i field_matched = false @@ -469,10 +491,10 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null break case TYPE_TIME: value, ok = cell.(time.Time) - // TODO convert to s/ms/us/ns?? if !ok { var intvalue int64 intvalue, ok = value.(int64) + // TODO convert to s/ms/us/ns?? if ok { value = time.Unix(intvalue, 0) } @@ -562,6 +584,28 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin return timestamp, measurement, err } } + // get dbname from row + if s.field_database_idx >= 0 { + dbname, err := s.GetStringFieldValue(s.field_database_idx) + if err != nil { + log.Printf("E! converting field dbname '%s'", s.column_name[s.field_database_idx]) + //cannot put data in correct, skip line + return timestamp, measurement, err + } else { + tags["dbname"] = dbname + } + } + // get host from row + if s.field_host_idx >= 0 { + host, err := s.GetStringFieldValue(s.field_host_idx) + if err != nil { + log.Printf("E! converting field host '%s'", s.column_name[s.field_host_idx]) + //cannot put data in correct, skip line + return timestamp, measurement, err + } else { + tags["host"] = host + } + } // fill tags for i := 0; i < s.tag_count; i++ { index := s.tag_idx[i] @@ -832,6 +876,10 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { } } + //import "reflect" + //// m1 and m2 are the maps we want to compare + //eq := reflect.DeepEqual(m1, m2) + acc.AddFields(measurement, fields, tags, timestamp) // fieldsG := map[string]interface{}{ @@ -842,7 +890,7 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { row_count += 1 } if Debug { - log.Printf("I! Query %d on %s found %d rows written, processing duration %s", q.index, p.Hosts[si], row_count, time.Since(query_time)) + log.Printf("I! Query %d found %d rows written, processing duration %s", q.index, row_count, time.Since(query_time)) } } } From 1c8554f807074a2273c3e1fd7587893ca96cdd90 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 25 May 2017 15:09:44 +0200 Subject: [PATCH 19/23] removed sqlite driver deps because of not cross compile on windows --- plugins/inputs/sql/README.md | 8 +-- plugins/inputs/sql/sql.go | 102 ++++++++++++++++++----------------- 2 files changed, 59 insertions(+), 51 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index a527a537276f7..811c0ea25e7d7 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -2,8 +2,8 @@ The plugin executes simple queries or query scripts on multiple servers. It permits to select the tags and the fields to export, if is needed fields can be forced to a choosen datatype. -Supported/integrated drivers are mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite) -Activable drivers (read below) are all golang SQL compliant drivers (see https://github.com/golang/go/wiki/SQLDrivers): for instance oci8 for Oracle +Supported/integrated drivers are mssql (SQLServer), mysql (MySQL), postgres (Postgres) +Activable drivers (read below) are all golang SQL compliant drivers (see https://github.com/golang/go/wiki/SQLDrivers): for instance oci8 for Oracle or sqlite3 (SQLite) ## Getting started : First you need to grant read/select privileges on queried tables to the database user you use for the connection @@ -26,8 +26,10 @@ import "C" import ( "log" - _ "github.com/mattn/go-oci8" // .. here you can add import to other drivers + _ "github.com/mattn/go-oci8" // requires external prorietary libs + // _ "bitbucket.org/phiggins/db2cli" // requires external prorietary libs + // _ "github.com/mattn/go-sqlite3" // not compiles on windows ) func main() { log.Printf("I! Loaded plugin of shared libs") diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 3944d46ac2164..7d7f866432566 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -19,14 +19,15 @@ import ( // database drivers here: _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" // pure go - _ "github.com/mattn/go-sqlite3" // _ "github.com/denisenkom/go-mssqldb" // pure go _ "github.com/zensqlmonitor/go-mssqldb" // pure go + // // the following commented because of the external proprietary libraries dependencies - // _ "github.com/mattn/go-oci8" - // _ "gopkg.in/rana/ora.v4" - // _ "bitbucket.org/phiggins/db2cli" // - // _ "github.com/SAP/go-hdb" + // _ "github.com/mattn/go-sqlite3" // builds only on linux + // _ "github.com/mattn/go-oci8" // requires external prorietary libs + // _ "gopkg.in/rana/ora.v4" // requires external prorietary libs + // _ "bitbucket.org/phiggins/db2cli" // requires external prorietary libs + // _ "github.com/SAP/go-hdb" // requires external prorietary libs // _ "github.com/a-palchikov/sqlago" ) @@ -40,6 +41,49 @@ const TYPE_AUTO = 0 var Debug = false var qindex = 0 +var sampleConfig = ` + [[inputs.sql]] + # debug=false # Enables very verbose output + + ## Database Driver + driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib + # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) + + ## Server DSNs + servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver + #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag + #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag + + ## Queries to perform (block below can be repeated) + [[inputs.sql.query]] + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # + measurement="connection_errors" # destination measurement + tag_cols=["application"] # colums used as tags + field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion + # + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer + # float_fields=["TEMPERATURE"] # adds fields and forces his value as float + # time_fields=[".*_TIME"] # adds fields and forces his value as time + # + # field_measurement = "CLASS" # the column that contains the name of the measurement + # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value + # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value + # field_name = "counter_name" # the column that contains the name of the counter + # field_value = "counter_value" # the column that contains the value of the counter + # + # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) + # + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) + null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) + sanitize = false # true: will perform some chars substitutions (false: use value as is) + ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) + ` + type Query struct { Query string Measurement string @@ -65,9 +109,10 @@ type Query struct { IgnoreOtherFields bool Sanitize bool IgnoreRowErrors bool + ParseTagsOnce bool // QueryScript string - Parameters []string //TODO + // Parameters []string //TODO // -------- internal data ----------- statements []*sql.Stmt @@ -91,6 +136,8 @@ type Query struct { field_value_type int field_timestamp_idx int + last_poll_ts time.Time + index int } @@ -144,48 +191,6 @@ func match_str(key string, str_array []string) bool { } func (s *Sql) SampleConfig() string { - var sampleConfig = ` - [[inputs.sql]] - # debug=false # Enables very verbose output - - ## Database Driver - driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] - # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib - # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - - ## Server DSNs - servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver - #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag - #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag - - ## Queries to perform (block below can be repeated) - [[inputs.sql.query]] - # query has precedence on query_script, if both query and query_script are defined only query is executed - query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" - # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file - # - measurement="connection_errors" # destination measurement - tag_cols=["application"] # colums used as tags - field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion - # - # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer - # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=[".*_TIME"] # adds fields and forces his value as time - # - # field_measurement = "CLASS" # the column that contains the name of the measurement - # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value - # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value - # field_name = "counter_name" # the column that contains the name of the counter - # field_value = "counter_value" # the column that contains the value of the counter - # - # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) - # - ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) - null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) - sanitize = false # true: will perform some chars substitutions (false: use value as is) - ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) - ` return sampleConfig } @@ -802,6 +807,7 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { log.Printf("I! Query %d exectution time: %s", q.index, time.Since(query_time)) } query_time = time.Now() + q.last_poll_ts = query_time if err != nil { return err From 580ba57f41f663dcdb6be280efa06d76005195f1 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Thu, 1 Jun 2017 15:53:14 +0200 Subject: [PATCH 20/23] removed commented code changed config sample toml indentation --- plugins/inputs/sql/sql.go | 205 ++++++++++---------------------------- 1 file changed, 50 insertions(+), 155 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 7d7f866432566..b4b134a57cfae 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -8,27 +8,17 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" "log" - "regexp" - // "net/url" "os" "plugin" "reflect" + "regexp" "strconv" "strings" "time" // database drivers here: _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" // pure go - // _ "github.com/denisenkom/go-mssqldb" // pure go - _ "github.com/zensqlmonitor/go-mssqldb" // pure go - // - // the following commented because of the external proprietary libraries dependencies - // _ "github.com/mattn/go-sqlite3" // builds only on linux - // _ "github.com/mattn/go-oci8" // requires external prorietary libs - // _ "gopkg.in/rana/ora.v4" // requires external prorietary libs - // _ "bitbucket.org/phiggins/db2cli" // requires external prorietary libs - // _ "github.com/SAP/go-hdb" // requires external prorietary libs - // _ "github.com/a-palchikov/sqlago" + _ "github.com/lib/pq" + _ "github.com/zensqlmonitor/go-mssqldb" ) const TYPE_STRING = 1 @@ -42,47 +32,47 @@ var Debug = false var qindex = 0 var sampleConfig = ` - [[inputs.sql]] - # debug=false # Enables very verbose output - - ## Database Driver - driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] - # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib - # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - - ## Server DSNs - servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver - #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag - #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag - - ## Queries to perform (block below can be repeated) - [[inputs.sql.query]] - # query has precedence on query_script, if both query and query_script are defined only query is executed - query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" - # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file - # - measurement="connection_errors" # destination measurement - tag_cols=["application"] # colums used as tags - field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion - # - # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer - # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=[".*_TIME"] # adds fields and forces his value as time - # - # field_measurement = "CLASS" # the column that contains the name of the measurement - # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value - # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value - # field_name = "counter_name" # the column that contains the name of the counter - # field_value = "counter_value" # the column that contains the value of the counter - # - # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) - # - ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) - null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) - sanitize = false # true: will perform some chars substitutions (false: use value as is) - ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) - ` +[[inputs.sql]] + # debug=false # Enables very verbose output + + ## Database Driver + driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib + # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) + + ## Server DSNs + servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver + #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag + #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag + + ## Queries to perform (block below can be repeated) + [[inputs.sql.query]] + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # + measurement="connection_errors" # destination measurement + tag_cols=["application"] # colums used as tags + field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion + # + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer + # float_fields=["TEMPERATURE"] # adds fields and forces his value as float + # time_fields=[".*_TIME"] # adds fields and forces his value as time + # + # field_measurement = "CLASS" # the column that contains the name of the measurement + # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value + # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value + # field_name = "counter_name" # the column that contains the name of the counter + # field_value = "counter_value" # the column that contains the value of the counter + # + # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) + # + ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) + null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) + sanitize = false # true: will perform some chars substitutions (false: use value as is) + ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) +` type Query struct { Query string @@ -112,7 +102,6 @@ type Query struct { ParseTagsOnce bool // QueryScript string - // Parameters []string //TODO // -------- internal data ----------- statements []*sql.Stmt @@ -122,11 +111,11 @@ type Query struct { cells []interface{} field_count int - field_idx []int //Column indexes of fields - field_type []int //Column types of fields + field_idx []int // Column indexes of fields + field_type []int // Column types of fields tag_count int - tag_idx []int //Column indexes of tags (strings) + tag_idx []int // Column indexes of tags (strings) field_host_idx int field_database_idx int @@ -203,44 +192,6 @@ type DSN struct { dbname string } -// -//func ParseDSN(dsn string) (*DSN, error) { -// -// url, err := url.Parse(dsn) -// if err != nil { -// return nil, err -// } -// pdsn := &DSN{} -// pdsn.host = url.Host -// pdsn.dbname = url.Path -// return pdsn, err -// -// res = map[string]string{} -// parts := strings.Split(dsn, ";") -// for _, part := range parts { -// if len(part) == 0 { -// continue -// } -// lst := strings.SplitN(part, "=", 2) -// name := strings.TrimSpace(strings.ToLower(lst[0])) -// if len(name) == 0 { -// continue -// } -// var value string = "" -// if len(lst) > 1 { -// value = strings.TrimSpace(lst[1]) -// } -// res[name] = value -// } -// return res -// // prm := &p.SessionPrm{Host: url.Host} -// // -// // if url.User != nil { -// // pdsn.Username = url.User.Username() -// // prm.Password, _ = url.User.Password() -// // } -//} - func (s *Sql) Init() error { Debug = s.Debug @@ -264,23 +215,7 @@ func (s *Sql) Init() error { s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) } } - // for i := 0; i < len(s.Servers); i++ { - // c, err := ParseDSN(s.Servers[i]) - // if err == nil { - // log.Printf("Host %s Database %s", c.host, c.dbname) - // } else { - // panic(err) - // } - // - // //TODO get host from server - // // mysql servers = ["nprobe:nprobe@tcp(neteye.wp.lan:3307)/nprobe"] - // // "postgres://nprobe:nprobe@rue-test/nprobe?sslmode=disable" - // // oracle telegraf/monitor@10.62.6.1:1522/tunapit - // // match, _ := regexp.MatchString(".*@([0-9.a-zA-Z]*)[:]?[0-9]*/.*", "peach") - // // fmt.Println(match) - // // addr, err := net.LookupHost("198.252.206.16") - // - // } + if len(s.Servers) > 0 && len(s.Hosts) > 0 && len(s.Hosts) != len(s.Servers) { return fmt.Errorf("For each server a host should be specified (%d/%d)", len(s.Hosts), len(s.Servers)) } @@ -298,7 +233,7 @@ func (s *Query) Init(cols []string) error { log.Printf("I! Init Query %d with %d columns", s.index, len(cols)) } - //Define index of tags and fields and keep it for reuse + // Define index of tags and fields and keep it for reuse s.column_name = cols // init the arrays for store row data @@ -306,14 +241,6 @@ func (s *Query) Init(cols []string) error { s.cells = make([]interface{}, col_count) s.cell_refs = make([]interface{}, col_count) - // init the arrays for store field/tag infos - // expected_tag_count := len(s.TagCols) - // var expected_field_count int - // if !s.IgnoreOtherFields { - // expected_field_count = col_count // - expected_tag_count - // } else { - // expected_field_count = len(s.FieldCols) + len(s.BoolFields) + len(s.IntFields) + len(s.FloatFields) + len(s.TimeFields) - // } // because of regex, now we must assume the max cols expected_field_count := col_count expected_tag_count := col_count @@ -364,23 +291,18 @@ func (s *Query) Init(cols []string) error { field_matched = false s.tag_idx[s.tag_count] = i s.tag_count++ - // cell = new(sql.RawBytes) cell = new(string) } else if match_str(s.column_name[i], s.IntFields) { dest_type = TYPE_INT cell = new(sql.RawBytes) - // cell = new(int); } else if match_str(s.column_name[i], s.FloatFields) { dest_type = TYPE_FLOAT - // cell = new(float64); cell = new(sql.RawBytes) } else if match_str(s.column_name[i], s.TimeFields) { dest_type = TYPE_TIME - // cell = new(string) cell = new(sql.RawBytes) } else if match_str(s.column_name[i], s.BoolFields) { dest_type = TYPE_BOOL - // cell = new(bool); cell = new(sql.RawBytes) } else if match_str(s.column_name[i], s.FieldCols) { dest_type = TYPE_AUTO @@ -388,7 +310,6 @@ func (s *Query) Init(cols []string) error { } else if !s.IgnoreOtherFields { dest_type = TYPE_AUTO cell = new(sql.RawBytes) - // cell = new(string); } else { field_matched = false cell = new(sql.RawBytes) @@ -499,8 +420,8 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null if !ok { var intvalue int64 intvalue, ok = value.(int64) - // TODO convert to s/ms/us/ns?? if ok { + // TODO convert to s/ms/us/ns?? value = time.Unix(intvalue, 0) } } @@ -535,9 +456,6 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null } } else { value = nil - if Debug { - // log.Printf("I! nil value for field name '%s' type %d", name, field_type) - } } if !ok { err = fmt.Errorf("Error by converting field %s", name) @@ -621,7 +539,6 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin // cannot put data in correct series, skip line return timestamp, measurement, err } else { - // log.Printf("******! tag %s=%s %s %s", name, value, , reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) tags[name] = value } } @@ -655,7 +572,6 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin if err != nil { // cannot get value of column with expected datatype, warning and continue log.Printf("W! converting value of field '%s'", name) - // return timestamp, measurement, err } else { fields[name] = value } @@ -737,7 +653,6 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err if err != nil { return nil, err } - //defer stmt.Close() } // execute prepared statement @@ -754,8 +669,6 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } } else { log.Printf("W! No query to execute %d", q.index) - // err = errors.New("No query to execute") - // return nil, err return nil, nil } @@ -844,15 +757,6 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { return err } - // for debug purposes... - // if row_count == 0 && Debug { - // for ci := 0; ci < len(q.cells); ci++ { - // if q.cells[ci] != nil { - // log.Printf("I! Column '%s' type %s, raw data '%s'", q.column_name[ci], reflect.TypeOf(q.cells[ci]).Kind(), fmt.Sprintf("%v", q.cells[ci])) - // } - // } - // } - // collect tags and fields tags := map[string]string{} fields := map[string]interface{}{} @@ -882,17 +786,8 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { } } - //import "reflect" - //// m1 and m2 are the maps we want to compare - //eq := reflect.DeepEqual(m1, m2) - acc.AddFields(measurement, fields, tags, timestamp) - // fieldsG := map[string]interface{}{ - // "usage_user": 100 * (cts.User - lastCts.User - (cts.Guest - lastCts.Guest)) / totalDelta, - // } - // acc.AddGauge("cpu", fieldsG, tags, now) // TODO use gauge too? - row_count += 1 } if Debug { From 348de99714abd3d7aa2601b7e236768c13b55804 Mon Sep 17 00:00:00 2001 From: lucadistefano Date: Mon, 24 Jul 2017 20:31:17 +0200 Subject: [PATCH 21/23] first pass reformatting help and readme, removed flags,renamed servers option into db_source_names and hosts into servers --- plugins/inputs/sql/README.md | 81 +++++---- plugins/inputs/sql/sql.go | 333 ++++++++++++++--------------------- 2 files changed, 171 insertions(+), 243 deletions(-) diff --git a/plugins/inputs/sql/README.md b/plugins/inputs/sql/README.md index 811c0ea25e7d7..372648395e761 100644 --- a/plugins/inputs/sql/README.md +++ b/plugins/inputs/sql/README.md @@ -55,48 +55,47 @@ The steps of above can be reused in a similar way for other proprietary and non ## Configuration: ``` - [[inputs.sql]] - # debug=false # Enables very verbose output - - ## Database Driver - driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] - # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib - # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - - ## Server DSNs - servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver - hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag - db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag - - ## Queries to perform (block below can be repeated) - [[inputs.sql.query]] - # query has precedence on query_script, if both query and query_script are defined only query is executed - query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" - # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file - # - measurement="connection_errors" # destination measurement - tag_cols=["application"] # colums used as tags - field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion - # - # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer - # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=[".*_TIME"] # adds fields and forces his value as time - # - # field_measurement = "CLASS" # the golumn that contains the name of the measurement - # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value - # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value - # field_name = "counter_name" # the column that contains the name of the counter - # field_value = "counter_value" # the column that contains the value of the counter - # - # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) - # - ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) - null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) - sanitize = false # true: will perform some chars substitutions (false: use value as is) - ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) + # debug=false # Enables very verbose output + + ## Database Driver + driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib + # keep_connection = false # optional: if true keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) + + ## Server DSNs + servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver + #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag + #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag + + ## Queries to perform (block below can be repeated) + [[inputs.sql.query]] + # query has precedence on query_script, if both query and query_script are defined only query is executed + query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + # + measurement="connection_errors" # destination measurement + tag_cols=["application"] # colums used as tags + field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion + # + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer + # float_fields=["TEMPERATURE"] # adds fields and forces his value as float + # time_fields=[".*_TIME"] # adds fields and forces his value as time + # + # field_measurement = "CLASS" # the column that contains the name of the measurement + # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value + # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value + # field_name = "counter_name" # the column that contains the name of the counter + # field_value = "counter_value" # the column that contains the value of the counter + # + # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) + # + # ignore_other_fields = false # optional: if query returns columns not defined, they are automatically added (true: ignore columns) + # sanitize = false # optional: will perform some chars substitutions (false: use value as is) + # null_as_zero = false # optional: converts null values into zero or empty strings (false: ignore fields) + # ignore_row_errors # optional: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) ``` -sql_script is read only once, if you change the script you need to restart telegraf +sql_script is read only once, if you change the script you need to reload telegraf ## Field names Field names are the same of the relative column name or taken from value of a column. If there is the need of rename the fields, just do it in the sql, try to use an ' AS ' . diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index b4b134a57cfae..ab0bc5955c4d8 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -17,7 +17,9 @@ import ( "time" // database drivers here: _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" + _ "github.com/jackc/pgx/stdlib" + // _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" // builds only on linux _ "github.com/zensqlmonitor/go-mssqldb" ) @@ -28,52 +30,8 @@ const TYPE_FLOAT = 4 const TYPE_TIME = 5 const TYPE_AUTO = 0 -var Debug = false var qindex = 0 -var sampleConfig = ` -[[inputs.sql]] - # debug=false # Enables very verbose output - - ## Database Driver - driver = "mysql" # required. Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] - # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" # optional: path to the golang 1.8 plugin shared lib - # keep_connection = false # true: keeps the connection with database instead to reconnect at each poll and uses prepared statements (false: reconnection at each poll, no prepared statements) - - ## Server DSNs - servers = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] # required. Connection DSN to pass to the DB driver - #hosts=["neteye", "hostmysql"] # optional: for each server a relative host entry should be specified and will be added as host tag - #db_names=["rue", "monitoring"] # optional: for each server a relative db name entry should be specified and will be added as dbname tag - - ## Queries to perform (block below can be repeated) - [[inputs.sql.query]] - # query has precedence on query_script, if both query and query_script are defined only query is executed - query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" - # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file - # - measurement="connection_errors" # destination measurement - tag_cols=["application"] # colums used as tags - field_cols=["avg_application_latency","avg_bytes","act_throughput"] # select fields and use the database driver automatic datatype conversion - # - # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer - # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=[".*_TIME"] # adds fields and forces his value as time - # - # field_measurement = "CLASS" # the column that contains the name of the measurement - # field_host = "DBHOST" # the column that contains the name of the database host used for host tag value - # field_database = "DBHOST" # the column that contains the name of the database used for dbname tag value - # field_name = "counter_name" # the column that contains the name of the counter - # field_value = "counter_value" # the column that contains the value of the counter - # - # field_timestamp = "sample_time" # the column where is to find the time of sample (should be a date datatype) - # - ignore_other_fields = false # false: if query returns columns not defined, they are automatically added (true: ignore columns) - null_as_zero = false # true: converts null values into zero or empty strings (false: ignore fields) - sanitize = false # true: will perform some chars substitutions (false: use value as is) - ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) -` - type Query struct { Query string Measurement string @@ -81,25 +39,21 @@ type Query struct { FieldTimestamp string TimestampUnit string // - TagCols []string - FieldCols []string - // + TagCols []string + // Data Conversion IntFields []string FloatFields []string BoolFields []string TimeFields []string - // + // Verical structure FieldHost string FieldName string FieldValue string FieldDatabase string FieldMeasurement string // - NullAsZero bool - IgnoreOtherFields bool - Sanitize bool - IgnoreRowErrors bool - ParseTagsOnce bool + Sanitize bool + IgnoreRowErrors bool // QueryScript string @@ -117,15 +71,17 @@ type Query struct { tag_count int tag_idx []int // Column indexes of tags (strings) + // Verical structure field_host_idx int field_database_idx int field_measurement_idx int field_name_idx int - field_value_idx int - field_value_type int field_timestamp_idx int + // Data Conversion + field_value_idx int + field_value_type int - last_poll_ts time.Time + // last_poll_ts time.Time index int } @@ -136,16 +92,13 @@ type Sql struct { KeepConnection bool - Servers []string + DbSourceNames []string - Hosts []string - DbNames []string + Servers []string Query []Query // internal - Debug bool - connections []*sql.DB initialized bool } @@ -180,47 +133,88 @@ func match_str(key string, str_array []string) bool { } func (s *Sql) SampleConfig() string { - return sampleConfig + return ` +[[inputs.sql]] + ## Database Driver, required. + ## Valid options: mssql (SQLServer), mysql (MySQL), postgres (Postgres), sqlite3 (SQLite), [oci8 ora.v4 (Oracle)] + driver = "mysql" + + ## optional: path to the golang 1.8 plugin shared lib where additional sql drivers are linked + # shared_lib = "/home/luca/.gocode/lib/oci8_go.so" + + ## optional: + ## if true keeps the connection with database instead to reconnect at each poll and uses prepared statements + ## if false reconnection at each poll, no prepared statements + # keep_connection = false + + ## Server DSNs, required. Connection DSN to pass to the DB driver + db_source_names = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] + + ## optional: for each server a relative host entry should be specified and will be added as server tag + #servers=["neteye", "hostmysql"] + + ## Queries to perform (block below can be repeated) + [[inputs.sql.query]] + ## query has precedence on query_script, if both query and query_script are defined only query is executed + query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" + # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file + + ## destination measurement + measurement="connection_errors" + ## colums used as tags + tag_cols=["application"] + ## select fields and use the database driver automatic datatype conversion + field_cols=["avg_application_latency","avg_bytes","act_throughput"] + + # + # bool_fields=["ON"] # adds fields and forces his value as bool + # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer + # float_fields=["TEMPERATURE"] # adds fields and forces his value as float + # time_fields=[".*_TIME"] # adds fields and forces his value as time + + ## Vertical srtucture + ## optional: the column that contains the name of the measurement, if not specified the value of the option measurement is used + # field_measurement = "CLASS" + ## the column that contains the name of the database host used for host tag value + # field_host = "DBHOST" + ## the column that contains the name of the database used for dbname tag value + # field_database = "DBHOST" + ## required if vertical: the column that contains the name of the counter + # field_name = "counter_name" + ## required if vertical: the column that contains the value of the counter + # field_value = "counter_value" + ## optional: the column where is to find the time of sample (should be a date datatype) + # field_timestamp = "sample_time" + # + #sanitize = false # true: will perform some chars substitutions (false: use value as is) + #ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) +` } func (_ *Sql) Description() string { return "SQL Plugin" } -type DSN struct { - host string - dbname string -} - func (s *Sql) Init() error { - Debug = s.Debug - - if Debug { - log.Printf("I! Init %d servers %d queries, driver %s", len(s.Servers), len(s.Query), s.Driver) - } + log.Printf("D! Init %d servers %d queries, driver %s", len(s.DbSourceNames), len(s.Query), s.Driver) if len(s.SharedLib) > 0 { _, err := plugin.Open(s.SharedLib) if err != nil { panic(err) } - if Debug { - log.Printf("I! Loaded shared lib '%s'", s.SharedLib) - } + log.Printf("D! Loaded shared lib '%s'", s.SharedLib) } if s.KeepConnection { - s.connections = make([]*sql.DB, len(s.Servers)) + s.connections = make([]*sql.DB, len(s.DbSourceNames)) for i := 0; i < len(s.Query); i++ { - s.Query[i].statements = make([]*sql.Stmt, len(s.Servers)) + s.Query[i].statements = make([]*sql.Stmt, len(s.DbSourceNames)) } } - if len(s.Servers) > 0 && len(s.Hosts) > 0 && len(s.Hosts) != len(s.Servers) { - return fmt.Errorf("For each server a host should be specified (%d/%d)", len(s.Hosts), len(s.Servers)) - } - if len(s.Servers) > 0 && len(s.DbNames) > 0 && len(s.DbNames) != len(s.Servers) { - return fmt.Errorf("For each server a db name should be specified (%d/%d)", len(s.DbNames), len(s.Servers)) + if len(s.DbSourceNames) > 0 && len(s.Servers) > 0 && len(s.Servers) != len(s.DbSourceNames) { + return fmt.Errorf("For each DSN a host should be specified (%d/%d)", len(s.Servers), len(s.DbSourceNames)) } return nil } @@ -229,9 +223,7 @@ func (s *Query) Init(cols []string) error { qindex++ s.index = qindex - if Debug { - log.Printf("I! Init Query %d with %d columns", s.index, len(cols)) - } + log.Printf("D! Init Query %d with %d columns", s.index, len(cols)) // Define index of tags and fields and keep it for reuse s.column_name = cols @@ -251,6 +243,7 @@ func (s *Query) Init(cols []string) error { s.tag_count = 0 s.field_count = 0 + // Vertical structure // prepare vars for vertical counter parsing s.field_name_idx = -1 s.field_value_idx = -1 @@ -280,6 +273,7 @@ func (s *Query) Init(cols []string) error { if (len(s.FieldValue) > 0 && len(s.FieldName) == 0) || (len(s.FieldName) > 0 && len(s.FieldValue) == 0) { return fmt.Errorf("Both field_name and field_value should be set") } + //------------ // fill columns info var cell interface{} @@ -292,6 +286,7 @@ func (s *Query) Init(cols []string) error { s.tag_idx[s.tag_count] = i s.tag_count++ cell = new(string) + // Datatype conversion } else if match_str(s.column_name[i], s.IntFields) { dest_type = TYPE_INT cell = new(sql.RawBytes) @@ -304,24 +299,13 @@ func (s *Query) Init(cols []string) error { } else if match_str(s.column_name[i], s.BoolFields) { dest_type = TYPE_BOOL cell = new(sql.RawBytes) - } else if match_str(s.column_name[i], s.FieldCols) { - dest_type = TYPE_AUTO - cell = new(sql.RawBytes) - } else if !s.IgnoreOtherFields { - dest_type = TYPE_AUTO - cell = new(sql.RawBytes) + // ------------- } else { - field_matched = false + dest_type = TYPE_AUTO cell = new(sql.RawBytes) - if Debug { - log.Printf("I! Skipped field %s", s.column_name[i]) - } - } - - if Debug && !field_matched { - log.Printf("I! Column %d '%s' dest type %d", i, s.column_name[i], dest_type) } + // Vertical structure if s.column_name[i] == s.FieldHost { s.field_host_idx = i field_matched = false @@ -347,6 +331,7 @@ func (s *Query) Init(cols []string) error { s.field_timestamp_idx = i field_matched = false } + //--------- if field_matched { s.field_type[s.field_count] = dest_type @@ -357,9 +342,7 @@ func (s *Query) Init(cols []string) error { s.cell_refs[i] = &s.cells[i] } - if Debug { - log.Printf("I! Query structure with %d tags and %d fields on %d columns...", s.tag_count, s.field_count, col_count) - } + log.Printf("D! Query structure with %d tags and %d fields on %d columns...", s.tag_count, s.field_count, col_count) return nil } @@ -375,9 +358,7 @@ func ConvertString(name string, cell interface{}) (string, bool) { if !ok { value = fmt.Sprintf("%v", cell) ok = true - if Debug { - log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) - } + log.Printf("W! converting '%s' type %s raw data '%s'", name, reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) } else { value = strconv.FormatInt(ivalue, 10) } @@ -388,7 +369,7 @@ func ConvertString(name string, cell interface{}) (string, bool) { return value, ok } -func (s *Query) ConvertField(name string, cell interface{}, field_type int, NullAsZero bool) (interface{}, error) { +func (s *Query) ConvertField(name string, cell interface{}, field_type int) (interface{}, error) { var value interface{} var ok bool var str string @@ -432,28 +413,6 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null default: value = cell } - } else if NullAsZero { - switch field_type { - case TYPE_AUTO: - case TYPE_STRING: - value = "" - break - case TYPE_INT: - value = 0i - case TYPE_FLOAT: - value = 0.0 - case TYPE_BOOL: - value = false - case TYPE_TIME: - value = time.Unix(0, 0) - break - default: - value = 0 - } - - if Debug { - log.Printf("I! forcing nil value of field '%s' type %d to %s", name, field_type, fmt.Sprintf("%v", value)) - } } else { value = nil } @@ -470,11 +429,7 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int, Null func (s *Query) GetStringFieldValue(index int) (string, error) { cell := s.cells[index] if cell == nil { - if s.NullAsZero { - return "", nil - } else { - return "", fmt.Errorf("Error converting name '%s' is nil", s.column_name[index]) - } + return "", fmt.Errorf("Error converting name '%s' is nil", s.column_name[index]) } value, ok := ConvertString(s.column_name[index], cell) @@ -489,9 +444,10 @@ func (s *Query) GetStringFieldValue(index int) (string, error) { } func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[string]string, fields map[string]interface{}) (time.Time, string, error) { + // Vertical structure if s.field_timestamp_idx >= 0 { // get the value of timestamp field - value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], s.cells[s.field_timestamp_idx], TYPE_TIME, false) + value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], s.cells[s.field_timestamp_idx], TYPE_TIME) if err != nil { return timestamp, measurement, errors.New("Cannot convert timestamp") } @@ -520,26 +476,13 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin } // get host from row if s.field_host_idx >= 0 { - host, err := s.GetStringFieldValue(s.field_host_idx) + server, err := s.GetStringFieldValue(s.field_host_idx) if err != nil { log.Printf("E! converting field host '%s'", s.column_name[s.field_host_idx]) //cannot put data in correct, skip line return timestamp, measurement, err } else { - tags["host"] = host - } - } - // fill tags - for i := 0; i < s.tag_count; i++ { - index := s.tag_idx[i] - name := s.column_name[index] - value, err := s.GetStringFieldValue(index) - if err != nil { - log.Printf("E! ignored tag %s", name) - // cannot put data in correct series, skip line - return timestamp, measurement, err - } else { - tags[name] = value + tags["server"] = server } } // vertical counters @@ -554,7 +497,7 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin // get the value of field var value interface{} - value, err = s.ConvertField(s.column_name[s.field_value_idx], s.cells[s.field_value_idx], s.field_value_type, s.NullAsZero) + value, err = s.ConvertField(s.column_name[s.field_value_idx], s.cells[s.field_value_idx], s.field_value_type) if err != nil { // cannot get value of column with expected datatype, skip line return timestamp, measurement, err @@ -563,12 +506,28 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin // fill the field fields[name] = value } + // --------------- + + // fill tags + for i := 0; i < s.tag_count; i++ { + index := s.tag_idx[i] + name := s.column_name[index] + value, err := s.GetStringFieldValue(index) + if err != nil { + log.Printf("E! ignored tag %s", name) + // cannot put data in correct series, skip line + return timestamp, measurement, err + } else { + tags[name] = value + } + } + // horizontal counters // fill fields from column values for i := 0; i < s.field_count; i++ { name := s.column_name[s.field_idx[i]] // get the value of field - value, err := s.ConvertField(name, s.cells[s.field_idx[i]], s.field_type[i], s.NullAsZero) + value, err := s.ConvertField(name, s.cells[s.field_idx[i]], s.field_type[i]) if err != nil { // cannot get value of column with expected datatype, warning and continue log.Printf("W! converting value of field '%s'", name) @@ -592,22 +551,16 @@ func (p *Sql) Connect(si int) (*sql.DB, error) { } if db == nil { - if Debug { - log.Printf("I! Setting up DB %s %s ...", p.Driver, p.Servers[si]) - } - db, err = sql.Open(p.Driver, p.Servers[si]) + log.Printf("D! Setting up DB %s %s ...", p.Driver, p.DbSourceNames[si]) + db, err = sql.Open(p.Driver, p.DbSourceNames[si]) if err != nil { return nil, err } } else { - if Debug { - log.Printf("I! Reusing connection to %s ...", p.Servers[si]) - } + log.Printf("D! Reusing connection to %s ...", p.DbSourceNames[si]) } - if Debug { - log.Printf("I! Connecting to DB %s ...", p.Servers[si]) - } + log.Printf("D! Connecting to DB %s ...", p.DbSourceNames[si]) err = db.Ping() if err != nil { return nil, err @@ -638,17 +591,13 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err buf := new(bytes.Buffer) buf.ReadFrom(filerc) q.Query = buf.String() - if Debug { - log.Printf("I! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) - } + log.Printf("D! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) } if len(q.Query) > 0 { if KeepConnection { // prepare statement if not already done if q.statements[si] == nil { - if Debug { - log.Printf("I! Preparing statement query %d...", q.index) - } + log.Printf("D! Preparing statement query %d...", q.index) q.statements[si], err = db.Prepare(q.Query) if err != nil { return nil, err @@ -656,15 +605,11 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err } // execute prepared statement - if Debug { - log.Printf("I! Performing query:\n\t\t%s\n...", q.Query) - } + log.Printf("D! Performing query:\n\t\t%s\n...", q.Query) rows, err = q.statements[si].Query() } else { // execute query - if Debug { - log.Printf("I! Performing query '%s'...", q.Query) - } + log.Printf("D! Performing query '%s'...", q.Query) rows, err = db.Query(q.Query) } } else { @@ -688,19 +633,15 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { p.initialized = true } - if Debug { - log.Printf("I! Starting poll") - } - for si := 0; si < len(p.Servers); si++ { + log.Printf("D! Starting poll") + for si := 0; si < len(p.DbSourceNames); si++ { var db *sql.DB var query_time time.Time db, err = p.Connect(si) query_time = time.Now() - if Debug { - duration := time.Since(query_time) - log.Printf("I! Server %d connection time: %s", si, duration) - } + duration := time.Since(query_time) + log.Printf("D! Server %d connection time: %s", si, duration) if err != nil { return err @@ -716,11 +657,10 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { query_time = time.Now() rows, err = q.Execute(db, si, p.KeepConnection) - if Debug { - log.Printf("I! Query %d exectution time: %s", q.index, time.Since(query_time)) - } + log.Printf("D! Query %d exectution time: %s", q.index, time.Since(query_time)) + query_time = time.Now() - q.last_poll_ts = query_time + // q.last_poll_ts = query_time if err != nil { return err @@ -763,17 +703,10 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { var measurement string // use database server as host, not the local host - if len(p.Hosts) > 0 { - _, ok := tags["host"] + if len(p.Servers) > 0 { + _, ok := tags["server"] if !ok { - tags["host"] = p.Hosts[si] - } - } - // add dbname tag - if len(p.DbNames) > 0 { - _, ok := tags["dbname"] - if !ok { - tags["dbname"] = p.DbNames[si] + tags["server"] = p.Servers[si] } } @@ -790,14 +723,10 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { row_count += 1 } - if Debug { - log.Printf("I! Query %d found %d rows written, processing duration %s", q.index, row_count, time.Since(query_time)) - } + log.Printf("D! Query %d found %d rows written, processing duration %s", q.index, row_count, time.Since(query_time)) } } - if Debug { - log.Printf("I! Poll done, duration %s", time.Since(start_time)) - } + log.Printf("D! Poll done, duration %s", time.Since(start_time)) return nil } From 263e2a5f5acf5b83c89fba852f397ef944f0aa11 Mon Sep 17 00:00:00 2001 From: Luca Di Stefano Date: Mon, 24 Jul 2017 23:32:33 +0200 Subject: [PATCH 22/23] removed ignore_row_errors flag removed breaks from case statement --- plugins/inputs/sql/sql.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index ab0bc5955c4d8..1b9285679ff8f 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -52,8 +52,7 @@ type Query struct { FieldDatabase string FieldMeasurement string // - Sanitize bool - IgnoreRowErrors bool + Sanitize bool // QueryScript string @@ -383,19 +382,16 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int) (int if ok { value, err = strconv.ParseInt(str, 10, 64) } - break case TYPE_FLOAT: str, ok = cell.(string) if ok { value, err = strconv.ParseFloat(str, 64) } - break case TYPE_BOOL: str, ok = cell.(string) if ok { value, err = strconv.ParseBool(str) } - break case TYPE_TIME: value, ok = cell.(time.Time) if !ok { @@ -406,10 +402,8 @@ func (s *Query) ConvertField(name string, cell interface{}, field_type int) (int value = time.Unix(intvalue, 0) } } - break case TYPE_STRING: value, ok = ConvertString(name, cell) - break default: value = cell } @@ -445,6 +439,7 @@ func (s *Query) GetStringFieldValue(index int) (string, error) { func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[string]string, fields map[string]interface{}) (time.Time, string, error) { // Vertical structure + // get timestamp from row if s.field_timestamp_idx >= 0 { // get the value of timestamp field value, err := s.ConvertField(s.column_name[s.field_timestamp_idx], s.cells[s.field_timestamp_idx], TYPE_TIME) @@ -474,7 +469,7 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin tags["dbname"] = dbname } } - // get host from row + // get server from row if s.field_host_idx >= 0 { server, err := s.GetStringFieldValue(s.field_host_idx) if err != nil { @@ -485,7 +480,7 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin tags["server"] = server } } - // vertical counters + // vertical counter if s.field_name_idx >= 0 { // get the name of the field name, err := s.GetStringFieldValue(s.field_name_idx) @@ -712,15 +707,10 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { timestamp, measurement, err = q.ParseRow(query_time, q.Measurement, tags, fields) if err != nil { - if q.IgnoreRowErrors { - log.Printf("W! Ignored error on row %d: %s", row_count, err) - } else { - return err - } + log.Printf("W! Ignored error on row %d: %s", row_count, err) + } else { + acc.AddFields(measurement, fields, tags, timestamp) } - - acc.AddFields(measurement, fields, tags, timestamp) - row_count += 1 } log.Printf("D! Query %d found %d rows written, processing duration %s", q.index, row_count, time.Since(query_time)) From 2b282eff9e7a9008ed61c11c1f30d66ee89e28b6 Mon Sep 17 00:00:00 2001 From: Luca Di Stefano Date: Fri, 7 Sep 2018 00:17:38 +0200 Subject: [PATCH 23/23] removed type conversion removed unused code removed multiple query removed sanitize added connection lifetime --- plugins/inputs/sql/sql.go | 307 +++++++++++++++++--------------------- 1 file changed, 140 insertions(+), 167 deletions(-) diff --git a/plugins/inputs/sql/sql.go b/plugins/inputs/sql/sql.go index 1b9285679ff8f..a36433a29a4dd 100644 --- a/plugins/inputs/sql/sql.go +++ b/plugins/inputs/sql/sql.go @@ -1,3 +1,25 @@ +// The MIT License (MIT) +// +// Copyright (c) 2016 Luca Di Stefano (luca@distefano.bz.it) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + package sql import ( @@ -30,22 +52,17 @@ const TYPE_FLOAT = 4 const TYPE_TIME = 5 const TYPE_AUTO = 0 -var qindex = 0 - type Query struct { Query string + QueryScript string Measurement string // FieldTimestamp string TimestampUnit string // TagCols []string - // Data Conversion - IntFields []string - FloatFields []string - BoolFields []string - TimeFields []string - // Verical structure + + // Vertical structure FieldHost string FieldName string FieldValue string @@ -54,15 +71,15 @@ type Query struct { // Sanitize bool // - QueryScript string // -------- internal data ----------- - statements []*sql.Stmt + statement *sql.Stmt column_name []string cell_refs []interface{} cells []interface{} + // Horizontal structure field_count int field_idx []int // Column indexes of fields field_type []int // Column types of fields @@ -70,36 +87,35 @@ type Query struct { tag_count int tag_idx []int // Column indexes of tags (strings) - // Verical structure + // Vertical structure field_host_idx int field_database_idx int field_measurement_idx int field_name_idx int field_timestamp_idx int + // Data Conversion field_value_idx int field_value_type int - - // last_poll_ts time.Time - - index int } type Sql struct { Driver string SharedLib string - KeepConnection bool - - DbSourceNames []string + //KeepConnection bool + MaxLifetime time.Duration - Servers []string + Source struct { + Dsn string + } Query []Query // internal - connections []*sql.DB - initialized bool + connection_ts time.Time + connection *sql.DB + initialized bool } var sanitizedChars = strings.NewReplacer("/sec", "_persec", "/Sec", "_persec", @@ -146,31 +162,28 @@ func (s *Sql) SampleConfig() string { ## if false reconnection at each poll, no prepared statements # keep_connection = false - ## Server DSNs, required. Connection DSN to pass to the DB driver - db_source_names = ["readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue", "readuser:sEcReT@tcp(hostmysql.wp.lan:3307)/monitoring"] + ## Maximum lifetime of a connection. + max_lifetime = "0s" - ## optional: for each server a relative host entry should be specified and will be added as server tag - #servers=["neteye", "hostmysql"] - + ## Connection information for data source. Table can be repeated to define multiple sources. + [[inputs.sql.source]] + ## Data source name for connecting. Syntax depends on selected driver. + dsn = "readuser:sEcReT@tcp(neteye.wp.lan:3307)/rue" + ## Queries to perform (block below can be repeated) [[inputs.sql.query]] ## query has precedence on query_script, if both query and query_script are defined only query is executed query="SELECT avg_application_latency,avg_bytes,act_throughput FROM Baselines WHERE application>0" # query_script = "/path/to/sql/script.sql" # if query is empty and a valid file is provided, the query will be read from file - ## destination measurement measurement="connection_errors" + + ## Horizontal srtucture ## colums used as tags tag_cols=["application"] ## select fields and use the database driver automatic datatype conversion field_cols=["avg_application_latency","avg_bytes","act_throughput"] - # - # bool_fields=["ON"] # adds fields and forces his value as bool - # int_fields=["MEMBERS",".*BYTES"] # adds fields and forces his value as integer - # float_fields=["TEMPERATURE"] # adds fields and forces his value as float - # time_fields=[".*_TIME"] # adds fields and forces his value as time - ## Vertical srtucture ## optional: the column that contains the name of the measurement, if not specified the value of the option measurement is used # field_measurement = "CLASS" @@ -183,10 +196,7 @@ func (s *Sql) SampleConfig() string { ## required if vertical: the column that contains the value of the counter # field_value = "counter_value" ## optional: the column where is to find the time of sample (should be a date datatype) - # field_timestamp = "sample_time" - # - #sanitize = false # true: will perform some chars substitutions (false: use value as is) - #ignore_row_errors # true: if an error in row parse is raised then the row will be skipped and the parse continue on next row (false: fatal error) + # field_timestamp = "sample_time" ` } @@ -195,7 +205,7 @@ func (_ *Sql) Description() string { } func (s *Sql) Init() error { - log.Printf("D! Init %d servers %d queries, driver %s", len(s.DbSourceNames), len(s.Query), s.Driver) + log.Printf("D! Init %s servers %d queries, driver %s", s.Source.Dsn, len(s.Query), s.Driver) if len(s.SharedLib) > 0 { _, err := plugin.Open(s.SharedLib) @@ -205,24 +215,11 @@ func (s *Sql) Init() error { log.Printf("D! Loaded shared lib '%s'", s.SharedLib) } - if s.KeepConnection { - s.connections = make([]*sql.DB, len(s.DbSourceNames)) - for i := 0; i < len(s.Query); i++ { - s.Query[i].statements = make([]*sql.Stmt, len(s.DbSourceNames)) - } - } - - if len(s.DbSourceNames) > 0 && len(s.Servers) > 0 && len(s.Servers) != len(s.DbSourceNames) { - return fmt.Errorf("For each DSN a host should be specified (%d/%d)", len(s.Servers), len(s.DbSourceNames)) - } return nil } func (s *Query) Init(cols []string) error { - qindex++ - s.index = qindex - - log.Printf("D! Init Query %d with %d columns", s.index, len(cols)) + log.Printf("D! Init Query with %d columns", len(cols)) // Define index of tags and fields and keep it for reuse s.column_name = cols @@ -261,13 +258,13 @@ func (s *Query) Init(cols []string) error { return fmt.Errorf("Missing column %s for given field_measurement", s.FieldMeasurement) } if len(s.FieldTimestamp) > 0 && !match_str(s.FieldTimestamp, s.column_name) { - return fmt.Errorf("Missing column %s for given field_measurement", s.FieldTimestamp) + return fmt.Errorf("Missing column %s for given field_timestamp", s.FieldTimestamp) } if len(s.FieldName) > 0 && !match_str(s.FieldName, s.column_name) { - return fmt.Errorf("Missing column %s for given field_measurement", s.FieldName) + return fmt.Errorf("Missing column %s for given field_name", s.FieldName) } if len(s.FieldValue) > 0 && !match_str(s.FieldValue, s.column_name) { - return fmt.Errorf("Missing column %s for given field_measurement", s.FieldValue) + return fmt.Errorf("Missing column %s for given field_value", s.FieldValue) } if (len(s.FieldValue) > 0 && len(s.FieldName) == 0) || (len(s.FieldName) > 0 && len(s.FieldValue) == 0) { return fmt.Errorf("Both field_name and field_value should be set") @@ -278,27 +275,13 @@ func (s *Query) Init(cols []string) error { var cell interface{} for i := 0; i < col_count; i++ { dest_type := TYPE_AUTO - field_matched := true + field_matched := true // is horizontal field if match_str(s.column_name[i], s.TagCols) { field_matched = false s.tag_idx[s.tag_count] = i s.tag_count++ cell = new(string) - // Datatype conversion - } else if match_str(s.column_name[i], s.IntFields) { - dest_type = TYPE_INT - cell = new(sql.RawBytes) - } else if match_str(s.column_name[i], s.FloatFields) { - dest_type = TYPE_FLOAT - cell = new(sql.RawBytes) - } else if match_str(s.column_name[i], s.TimeFields) { - dest_type = TYPE_TIME - cell = new(sql.RawBytes) - } else if match_str(s.column_name[i], s.BoolFields) { - dest_type = TYPE_BOOL - cell = new(sql.RawBytes) - // ------------- } else { dest_type = TYPE_AUTO cell = new(sql.RawBytes) @@ -308,35 +291,32 @@ func (s *Query) Init(cols []string) error { if s.column_name[i] == s.FieldHost { s.field_host_idx = i field_matched = false - } - if s.column_name[i] == s.FieldDatabase { + } else if s.column_name[i] == s.FieldDatabase { s.field_database_idx = i field_matched = false - } - if s.column_name[i] == s.FieldMeasurement { + } else if s.column_name[i] == s.FieldMeasurement { s.field_measurement_idx = i field_matched = false - } - if s.column_name[i] == s.FieldName { + } else if s.column_name[i] == s.FieldName { s.field_name_idx = i field_matched = false - } - if s.column_name[i] == s.FieldValue { + } else if s.column_name[i] == s.FieldValue { s.field_value_idx = i s.field_value_type = dest_type field_matched = false - } - if s.column_name[i] == s.FieldTimestamp { + } else if s.column_name[i] == s.FieldTimestamp { s.field_timestamp_idx = i field_matched = false } - //--------- + // Horizontal if field_matched { s.field_type[s.field_count] = dest_type s.field_idx[s.field_count] = i s.field_count++ } + + // s.cells[i] = cell s.cell_refs[i] = &s.cells[i] } @@ -431,14 +411,15 @@ func (s *Query) GetStringFieldValue(index int) (string, error) { return "", fmt.Errorf("Error converting name '%s' type %s, raw data '%s'", s.column_name[index], reflect.TypeOf(cell).Kind(), fmt.Sprintf("%v", cell)) } - if s.Sanitize { - value = sanitize(value) - } + // if s.Sanitize { + // value = sanitize(value) + // } return value, nil } func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[string]string, fields map[string]interface{}) (time.Time, string, error) { // Vertical structure + // get timestamp from row if s.field_timestamp_idx >= 0 { // get the value of timestamp field @@ -534,40 +515,41 @@ func (s *Query) ParseRow(timestamp time.Time, measurement string, tags map[strin return timestamp, measurement, nil } -func (p *Sql) Connect(si int) (*sql.DB, error) { +func (p *Sql) Connect() (*sql.DB, error) { var err error // create connection to db server if not already done var db *sql.DB - if p.KeepConnection { - db = p.connections[si] + if p.MaxLifetime > 0 && time.Since(p.connection_ts) < p.MaxLifetime { + db = p.connection } else { db = nil } if db == nil { - log.Printf("D! Setting up DB %s %s ...", p.Driver, p.DbSourceNames[si]) - db, err = sql.Open(p.Driver, p.DbSourceNames[si]) + log.Printf("D! Setting up DB %s %s ...", p.Driver, p.Source.Dsn) + db, err = sql.Open(p.Driver, p.Source.Dsn) if err != nil { return nil, err } + p.connection_ts = time.Now() } else { - log.Printf("D! Reusing connection to %s ...", p.DbSourceNames[si]) + log.Printf("D! Reusing connection to %s ...", p.Source.Dsn) } - log.Printf("D! Connecting to DB %s ...", p.DbSourceNames[si]) + log.Printf("D! Connecting to DB %s ...", p.Source.Dsn) err = db.Ping() if err != nil { return nil, err } - if p.KeepConnection { - p.connections[si] = db + if p.MaxLifetime > 0 { + p.connection = db } return db, nil } -func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, error) { +func (q *Query) Execute(db *sql.DB, KeepConnection bool) (*sql.Rows, error) { var err error var rows *sql.Rows // read query from sql script and put it in query string @@ -586,14 +568,14 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err buf := new(bytes.Buffer) buf.ReadFrom(filerc) q.Query = buf.String() - log.Printf("D! Read %d bytes SQL script from '%s' for query %d ...", len(q.Query), q.QueryScript, q.index) + log.Printf("D! Read %d bytes SQL script from '%s' for query ...", len(q.Query), q.QueryScript) } if len(q.Query) > 0 { if KeepConnection { // prepare statement if not already done - if q.statements[si] == nil { - log.Printf("D! Preparing statement query %d...", q.index) - q.statements[si], err = db.Prepare(q.Query) + if q.statement == nil { + log.Printf("D! Preparing statement query ...") + q.statement, err = db.Prepare(q.Query) if err != nil { return nil, err } @@ -601,14 +583,14 @@ func (q *Query) Execute(db *sql.DB, si int, KeepConnection bool) (*sql.Rows, err // execute prepared statement log.Printf("D! Performing query:\n\t\t%s\n...", q.Query) - rows, err = q.statements[si].Query() + rows, err = q.statement.Query() } else { // execute query log.Printf("D! Performing query '%s'...", q.Query) rows, err = db.Query(q.Query) } } else { - log.Printf("W! No query to execute %d", q.index) + log.Printf("W! No query to execute") return nil, nil } @@ -629,93 +611,84 @@ func (p *Sql) Gather(acc telegraf.Accumulator) error { } log.Printf("D! Starting poll") - for si := 0; si < len(p.DbSourceNames); si++ { - var db *sql.DB - var query_time time.Time - db, err = p.Connect(si) + var db *sql.DB + var query_time time.Time + + db, err = p.Connect() + query_time = time.Now() + duration := time.Since(query_time) + log.Printf("D! Server %s connection time: %s", p.Source.Dsn, duration) + + if err != nil { + return err + } + if p.MaxLifetime == 0 { + defer db.Close() + } + + // execute queries + for qi := 0; qi < len(p.Query); qi++ { + var rows *sql.Rows + q := &p.Query[qi] + + query_time = time.Now() + rows, err = q.Execute(db, p.MaxLifetime > 0) + log.Printf("D! Query exectution time: %s", time.Since(query_time)) + query_time = time.Now() - duration := time.Since(query_time) - log.Printf("D! Server %d connection time: %s", si, duration) if err != nil { return err } - if !p.KeepConnection { - defer db.Close() + if rows == nil { + continue } + defer rows.Close() - // execute queries - for qi := 0; qi < len(p.Query); qi++ { - var rows *sql.Rows - q := &p.Query[qi] - - query_time = time.Now() - rows, err = q.Execute(db, si, p.KeepConnection) - log.Printf("D! Query %d exectution time: %s", q.index, time.Since(query_time)) - - query_time = time.Now() - // q.last_poll_ts = query_time - + if q.field_count == 0 { + // initialize once the structure of query + var cols []string + cols, err = rows.Columns() if err != nil { return err } - if rows == nil { - continue - } - defer rows.Close() - - if q.field_count == 0 { - // initialize once the structure of query - var cols []string - cols, err = rows.Columns() - if err != nil { - return err - } - err = q.Init(cols) - if err != nil { - return err - } + err = q.Init(cols) + if err != nil { + return err } + } - row_count := 0 + row_count := 0 - for rows.Next() { - var timestamp time.Time + for rows.Next() { + var timestamp time.Time - if err = rows.Err(); err != nil { - return err - } - // database driver datatype conversion - err := rows.Scan(q.cell_refs...) - if err != nil { - return err - } + if err = rows.Err(); err != nil { + return err + } + // database driver datatype conversion + err := rows.Scan(q.cell_refs...) + if err != nil { + return err + } - // collect tags and fields - tags := map[string]string{} - fields := map[string]interface{}{} - var measurement string - - // use database server as host, not the local host - if len(p.Servers) > 0 { - _, ok := tags["server"] - if !ok { - tags["server"] = p.Servers[si] - } - } + // collect tags and fields + tags := map[string]string{} + fields := map[string]interface{}{} + var measurement string - timestamp, measurement, err = q.ParseRow(query_time, q.Measurement, tags, fields) - if err != nil { - log.Printf("W! Ignored error on row %d: %s", row_count, err) - } else { - acc.AddFields(measurement, fields, tags, timestamp) - } - row_count += 1 + timestamp, measurement, err = q.ParseRow(query_time, q.Measurement, tags, fields) + if err != nil { + log.Printf("W! Ignored error on row %d: %s", row_count, err) + } else { + acc.AddFields(measurement, fields, tags, timestamp) } - log.Printf("D! Query %d found %d rows written, processing duration %s", q.index, row_count, time.Since(query_time)) + row_count += 1 } + log.Printf("D! Query found %d rows written, processing duration %s", row_count, time.Since(query_time)) } + log.Printf("D! Poll done, duration %s", time.Since(start_time)) return nil