Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding HAProxy module #2384

Merged
merged 23 commits into from
Sep 8, 2016
Merged
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
8c236c4
Merge pull request #1 from elastic/master
hartfordfive Jun 15, 2016
5b4adac
Merge pull request #2 from elastic/master
hartfordfive Jul 27, 2016
056f7f8
Committing current work for haproxy module
hartfordfive Aug 11, 2016
7b4c97d
Committing current changes.
hartfordfive Aug 13, 2016
8d9d696
Added haproxy package to fetch stat/info commands
hartfordfive Aug 16, 2016
dcf16a8
Updated stat/info metrics
hartfordfive Aug 16, 2016
16dafbe
Comitting current changes
hartfordfive Aug 21, 2016
ac6dbe1
Committing current changes
hartfordfive Aug 21, 2016
543319e
Updated necessary test related files
hartfordfive Aug 21, 2016
52652bf
Committing current changes
hartfordfive Aug 24, 2016
eb7c22d
Committing current changes
hartfordfive Aug 24, 2016
6c0eb6f
Fixed errors & small changes
hartfordfive Aug 27, 2016
e2828a8
Minor fixes
hartfordfive Aug 27, 2016
3e4384e
Fixed field names/values & removed unused function
hartfordfive Aug 29, 2016
004d9fa
Updated filebeat configs
hartfordfive Aug 29, 2016
007d4a4
Comitting curent changes
hartfordfive Aug 31, 2016
ef9c877
Merging in master & fixing conflicts
hartfordfive Sep 1, 2016
498ada4
Applied changes as per comments
hartfordfive Sep 2, 2016
e989718
Updated structure to follow event convetions
hartfordfive Sep 3, 2016
41eb385
Updated index templates
hartfordfive Sep 4, 2016
607c564
Fixed index templates and data type
hartfordfive Sep 4, 2016
6ba4fbd
Removed unecessary Sprintf & added experimental warning
hartfordfive Sep 5, 2016
e768cd2
Restored original version of file
hartfordfive Sep 7, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixed errors & small changes
  • Loading branch information
hartfordfive committed Aug 27, 2016
commit 6c0eb6f5df8e6844fd495484f670d6e561428596
1 change: 1 addition & 0 deletions metricbeat/.gitignore
Original file line number Diff line number Diff line change
@@ -3,3 +3,4 @@ build
/metricbeat
/metricbeat.test
/docs/html_docs
/logs/*
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be not needed anymore as soon as this is rebased on top of master or master is merged.

2 changes: 1 addition & 1 deletion metricbeat/module/haproxy/_meta/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
FROM haproxy:1.6
COPY haproxy.conf /usr/local/etc/haproxy/haproxy.cfg
COPY ./haproxy.conf /usr/local/etc/haproxy/haproxy.cfg
18 changes: 10 additions & 8 deletions metricbeat/module/haproxy/haproxy.go
Original file line number Diff line number Diff line change
@@ -132,7 +132,7 @@ type Info struct {

// Client is an instance of the HAProxy client
type Client struct {
connection net.Conn
//connection net.Conn
Address string
ProtoScheme string
}
@@ -163,16 +163,17 @@ func (c *Client) run(cmd string) (*bytes.Buffer, error) {
if err != nil {
return response, err
}
c.connection = conn
//c.connection = conn

defer c.connection.Close()
//defer c.connection.Close()
defer conn.Close()

_, err = c.connection.Write([]byte(cmd + "\n"))
_, err = conn.Write([]byte(cmd + "\n"))
if err != nil {
return response, err
}

_, err = io.Copy(response, c.connection)
_, err = io.Copy(response, conn)
if err != nil {
return response, err
}
@@ -185,13 +186,14 @@ func (c *Client) run(cmd string) (*bytes.Buffer, error) {
}

// GetStat returns the result from the 'show stat' command
func (c *Client) GetStat() (statRes []*Stat, err error) {
func (c *Client) GetStat() ([]*Stat, error) {

runResult, err := c.run("show stat")
if err != nil {
return nil, err
}

var statRes []*Stat
csvReader := csv.NewReader(runResult)
csvReader.TrailingComma = true

@@ -205,7 +207,7 @@ func (c *Client) GetStat() (statRes []*Stat, err error) {
}

// GetInfo returns the result from the 'show stat' command
func (c *Client) GetInfo() (infoRes *Info, err error) {
func (c *Client) GetInfo() (*Info, error) {

res, err := c.run("show info")
if err != nil {
@@ -234,7 +236,7 @@ func (c *Client) GetInfo() (infoRes *Info, err error) {
var result *Info
err := mapstructure.Decode(resultMap, &result)
if err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider writing this which is slightly more concise. (not a mandatory change, it's just how I'd of written it)

if err := mapstructure.Decode(resultMap, &result); err != nil {
    return nil, err
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I definitely agree it's cleaner that way, I'll definitely change that.

panic(err)
return nil, err
}
return result, nil
}
94 changes: 48 additions & 46 deletions metricbeat/module/haproxy/info/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -4,191 +4,193 @@
General infomration collected on HAProxy process
fields:
- name: nb_proc
type: intger
type: integer
description: >
Number of processes

- name: process_num
type: intger
type: integer
description: >
Process number

- name: pid
type: intger
type: integer
description: >
Process ID

- name: uptime_sec
type: intger
type: integer
description: >
Current uptime in seconds

- name: mem_max_mb
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider reporting this field as bytes rather than MB. Then you can apply a bytes field formatter so that Kibana renders the value nicely. If you do change it to bytes, then add format: bytes to this field. See this for an example: https://github.com/elastic/beats/blob/master/metricbeat/module/system/memory/_meta/fields.yml#L8

type: intger
type: integer
format: bytes
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the value is actually in MB then you don't want to format it as bytes. I recommend converting the value to bytes and renaming the field so you can take advantage of the byte formatter in Kibana (it uses http://numeraljs.com/).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I see you do convert the value. So then I think the field just needs a rename to reflect the correct units.

description: >
Max number of memory usage in MB

- name: ulimit_n
type: intger
type: integer
description: >
Max number of open files for process

- name: max_sock
type: intger
type: integer
description: >

- name: max_conn
type: intger
type: integer
description: >

- name: hard_max_conn
type: intger
type: integer
description: >

- name: curr_conns
type: intger
type: integer
description: >

- name: cum_conns
type: intger
type: integer
description: >

- name: cum_req
type: intger
type: integer
description: >

- name: max_ssl_conns
type: intger
type: integer
description: >

- name: curr_ssl_conns
type: intger
type: integer
description: >

- name: cum_ssl_conns
type: intger
type: integer
description: >

- name: max_pipes
type: intger
type: integer
description: >

- name: pipes_used
type: intger
type: integer
description: >

- name: pipes_free
type: intger
type: integer
description: >

- name: conn_rate
type: intger
type: integer
description: >

- name: conn_rate_limit
type: intger
type: integer
description: >

- name: max_conn_rate
type: intger
type: integer
description: >

- name: sess_rate
type: intger
type: integer
description: >

- name: sess_rate_limit
type: intger
type: integer
description: >

- name: max_sess_rate
type: intger
type: integer
description: >

- name: ssl_rate
type: intger
type: integer
description: >

- name: ssl_rate_limit
type: intger
type: integer
description: >

- name: max_ssl_rate
type: intger
type: integer
description: >

- name: ssl_frontend_key_rate
type: intger
type: integer
description: >

- name: ssl_frontend_max_key_rate
type: intger
type: integer
description: >

- name: ssl_frontend_session_reuse_pct
type: intger
type: integer
description: >

- name: ssl_babckend_key_rate
type: intger
type: integer
description: >

- name: ssl_frontend_key_rate
type: intger
type: integer
description: >

- name: ssl_frontend_max_key_rate
type: intger
type: integer
description: >

- name: ssl_frontend_session_reuse_pct
type: intger
type: integer
description: >

- name: ssl_babckend_key_rate
type: intger
type: integer
description: >

- name: ssl_backend_max_key_rate
type: intger
type: integer
description: >

- name: ssl_cached_lookups
type: intger
type: integer
description: >

- name: ssl_cache_misses
type: intger
type: integer
description: >

- name: compress_bps_in
type: intger
type: integer
description: >

- name: compress_bps_out
type: intger
type: integer
description: >

- name: compress_bps_rate_limit
type: intger
type: integer
description: >

- name: zlib_mem_usage
type: intger
type: integer
description: >

- name: max_zlib_mem_usage
type: intger
type: integer
description: >

- name: tasks
type: intger
type: integer
description: >

- name: run_queue
type: intger
type: integer
description: >

- name: idle_pct
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider making this value range on [0, 1] so you can use a float with a percentage formatter. For example: https://github.com/elastic/beats/blob/master/metricbeat/module/system/memory/_meta/fields.yml#L26-L27

type: intger
type: scaled_float
format: percent
description: >
16 changes: 14 additions & 2 deletions metricbeat/module/haproxy/info/data.go
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ import (
s "github.com/elastic/beats/metricbeat/schema"
c "github.com/elastic/beats/metricbeat/schema/mapstrstr"
"reflect"
"strconv"
"strings"
)

@@ -52,7 +53,7 @@ var (
"max_zlib_mem_usage": c.Int("MaxZlibMemUsage"),
"tasks": c.Int("Tasks"),
"run_queue": c.Int("Run_queue"),
"idle_pct": c.Int("Idle_pct"),
"idle_pct": c.Float("Idle_pct"),
}
)

@@ -76,7 +77,18 @@ func parseResponse(data []byte) map[string]string {
if parts[0] == "Name" || parts[0] == "Version" || parts[0] == "Release_date" || parts[0] == "Uptime" || parts[0] == "node" || parts[0] == "description" {
continue
}
resultMap[parts[0]] = strings.Trim(parts[1], " ")

if parts[0] == "Idle_pct" {
// Convert this value to a float between 0.0 and 1.0
f, _ := strconv.ParseFloat(parts[1], 64)
resultMap[parts[0]] = strconv.FormatFloat(f/float64(100), 'f', 2, 64)
} else if parts[0] == "Memmax_MB" {
// Convert this value to bytes
val, _ := strconv.Atoi(strings.Trim(parts[1], " "))
resultMap[parts[0]] = strconv.Itoa((val * 1024 * 1024))
} else {
resultMap[parts[0]] = strings.Trim(parts[1], " ")
}
}
return resultMap
}
4 changes: 2 additions & 2 deletions metricbeat/module/haproxy/info/info.go
Original file line number Diff line number Diff line change
@@ -65,13 +65,13 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {

hapc, err := haproxy.NewHaproxyClient(m.statsAddr)
if err != nil {
return nil, fmt.Errorf(fmt.Sprintf("HAProxy Client error: %s", err))
return nil, fmt.Errorf("HAProxy Client error: %s", err)
}

res, err := hapc.GetInfo()

if err != nil {
return nil, fmt.Errorf(fmt.Sprintf("HAProxy Client error fetching %s: %s", statsMethod, err))
return nil, fmt.Errorf("HAProxy Client error fetching %s: %s", statsMethod, err)
}
m.counter++