Allow specifying custom field names

README.md

@@ -57,19 +57,25 @@ VERSION:
    0.5.0
 
 AUTHOR:
-  Antoine Grondin - <antoine@digitalocean.com>
+   Antoine Grondin - <antoine@digitalocean.com>
 
 COMMANDS:
    help, h  Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
-   --skip '--skip option --skip option'   keys to skip when parsing a log entry
-   --keep '--keep option --keep option'   keys to keep when parsing a log entry
-   --sort-longest       sort by longest key after having sorted lexicographically
-   --skip-unchanged        skip keys that have the same value than the previous entry
-   --truncate           truncates values that are longer than --truncate-length
-   --truncate-length '15'     truncate values that are longer than this length
-   --help, -h           show help
-   --version, -v        print the version
+   --skip value                      keys to skip when parsing a log entry
+   --keep value                      keys to keep when parsing a log entry
+   --sort-longest                    sort by longest key after having sorted lexicographically
+   --skip-unchanged                  skip keys that have the same value than the previous entry
+   --truncate                        truncates values that are longer than --truncate-length
+   --truncate-length value           truncate values that are longer than this length (default: 15)
+   --light-bg                        use black as the base foreground color (for terminals with light backgrounds)
+   --time-format value               output time format, see https://golang.org/pkg/time/ for details (default: "Jan _2 15:04:05")
+   --ignore-interrupts, -i           ignore interrupts
+   --message-fields value, -m value  Custom JSON fields to search for the log message. (i.e. mssge, data.body.message) (default: "data.message") [$HUMANLOG_MESSAGE_FIELDS]
+   --time-fields value, -t value     Custom JSON fields to search for the log time. (i.e. logtime, data.body.datetime) [$HUMANLOG_TIME_FIELDS]
+   --level-fields value, -l value    Custom JSON fields to search for the log level. (i.e. somelevel, data.level) [$HUMANLOG_LEVEL_FIELDS]
+   --help, -h                        show help
+   --version, -v                     print the version
 ```
 [l2met]: https://github.com/ryandotsmith/l2met

cmd/humanlog/main.go

@@ -88,14 +88,38 @@ func newApp() *cli.App {
 		Usage: "ignore interrupts",
 	}
 
+	messageFields := cli.StringSlice{}
+	messageFieldsFlag := cli.StringSliceFlag{
+		Name:   "message-fields, m",
+		Usage:  "Custom JSON fields to search for the log message. (i.e. mssge, data.body.message)",
+		EnvVar: "HUMANLOG_MESSAGE_FIELDS",
+		Value:  &messageFields,
+	}
+
+	timeFields := cli.StringSlice{}
+	timeFieldsFlag := cli.StringSliceFlag{
+		Name:   "time-fields, t",
+		Usage:  "Custom JSON fields to search for the log time. (i.e. logtime, data.body.datetime)",
+		EnvVar: "HUMANLOG_TIME_FIELDS",
+		Value:  &timeFields,
+	}
+
+	levelFields := cli.StringSlice{}
+	levelFieldsFlag := cli.StringSliceFlag{
+		Name:   "level-fields, l",
+		Usage:  "Custom JSON fields to search for the log level. (i.e. somelevel, data.level)",
+		EnvVar: "HUMANLOG_LEVEL_FIELDS",
+		Value:  &levelFields,
+	}
+
 	app := cli.NewApp()
 	app.Author = "Antoine Grondin"
 	app.Email = "antoinegrondin@gmail.com"
 	app.Name = "humanlog"
 	app.Version = Version
 	app.Usage = "reads structured logs from stdin, makes them pretty on stdout!"
 
-	app.Flags = []cli.Flag{skipFlag, keepFlag, sortLongest, skipUnchanged, truncates, truncateLength, lightBg, timeFormat, ignoreInterrupts}
+	app.Flags = []cli.Flag{skipFlag, keepFlag, sortLongest, skipUnchanged, truncates, truncateLength, lightBg, timeFormat, ignoreInterrupts, messageFieldsFlag, timeFieldsFlag, levelFieldsFlag}
 
 	app.Action = func(c *cli.Context) error {
 
@@ -116,6 +140,18 @@ func newApp() *cli.App {
 			opts.SetKeep(keep)
 		}
 
+		if c.IsSet(strings.Split(messageFieldsFlag.Name, ",")[0]) {
+			opts.MessageFields = messageFields
+		}
+
+		if c.IsSet(strings.Split(timeFieldsFlag.Name, ",")[0]) {
+			opts.TimeFields = timeFields
+		}
+
+		if c.IsSet(strings.Split(levelFieldsFlag.Name, ",")[0]) {
+			opts.LevelFields = levelFields
+		}
+
 		if c.IsSet(strings.Split(ignoreInterrupts.Name, ",")[0]) {
 			signal.Ignore(os.Interrupt)
 		}

handler.go

@@ -22,6 +22,10 @@ var DefaultOptions = &HandlerOptions{
 	TruncateLength: 15,
 	TimeFormat:     time.Stamp,
 
+	TimeFields:    []string{"time", "ts", "@timestamp", "timestamp"},
+	MessageFields: []string{"message", "msg"},
+	LevelFields:   []string{"level", "lvl", "loglevel", "severity"},
+
 	KeyColor:              color.New(color.FgGreen),
 	ValColor:              color.New(color.FgHiWhite),
 	TimeLightBgColor:      color.New(color.FgBlack),
@@ -40,8 +44,13 @@ var DefaultOptions = &HandlerOptions{
 }
 
 type HandlerOptions struct {
-	Skip           map[string]struct{}
-	Keep           map[string]struct{}
+	Skip map[string]struct{}
+	Keep map[string]struct{}
+
+	TimeFields    []string
+	MessageFields []string
+	LevelFields   []string
+
 	SortLongest    bool
 	SkipUnchanged  bool
 	Truncates      bool

json_handler.go

@@ -13,7 +13,7 @@ import (
 	"github.com/fatih/color"
 )
 
-// JSONHandler can handle logs emmited by logrus.TextFormatter loggers.
+// JSONHandler can handle logs emitted by logrus.TextFormatter loggers.
 type JSONHandler struct {
 	buf     *bytes.Buffer
 	out     *tabwriter.Writer
@@ -29,6 +29,35 @@ type JSONHandler struct {
 	last map[string]string
 }
 
+// searchJSON searches a document for a key using the found func to determine if the value is accepted.
+// kvs is the deserialized json document.
+// fieldList is a list of field names that should be searched. Sub-documents can be searched by using the dot (.). For example, to search {"data"{"message": "<this field>"}} the item would be data.message
+func searchJSON(kvs map[string]interface{}, fieldList []string, found func(key string, value interface{}) bool) bool {
+	for _, field := range fieldList {
+		splits := strings.SplitN(field, ".", 2)
+		if len(splits) > 1 {
+			name, fieldKey := splits[0], splits[1]
+			val, ok := kvs[name]
+			if !ok {
+				// the key does not exist in the document
+				continue
+			}
+			if m, ok := val.(map[string]interface{}); ok {
+				// its value is JSON and was unmarshaled to map[string]interface{} so search the sub document
+				return searchJSON(m, []string{fieldKey}, found)
+			}
+		} else {
+			// this is not a sub-document search, so search the root
+			for k, v := range kvs {
+				if field == k && found(k, v) {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
 func checkEachUntilFound(fieldList []string, found func(string) bool) bool {
 	for _, field := range fieldList {
 		if found(field) {
@@ -38,15 +67,6 @@ func checkEachUntilFound(fieldList []string, found func(string) bool) bool {
 	return false
 }
 
-// supportedTimeFields enumerates supported timestamp field names
-var supportedTimeFields = []string{"time", "ts", "@timestamp", "timestamp"}
-
-// supportedMessageFields enumarates supported Message field names
-var supportedMessageFields = []string{"message", "msg"}
-
-// supportedLevelFields enumarates supported level field names
-var supportedLevelFields = []string{"level", "lvl", "loglevel", "severity"}
-
 func (h *JSONHandler) clear() {
 	h.Level = ""
 	h.Time = time.Time{}
@@ -67,6 +87,28 @@ func (h *JSONHandler) TryHandle(d []byte) bool {
 	return true
 }
 
+func deleteJSONKey(key string, jsonData map[string]interface{}) {
+	if _, ok := jsonData[key]; ok {
+		// found the key at the root
+		delete(jsonData, key)
+		return
+	}
+
+	splits := strings.SplitN(key, ".", 2)
+	if len(splits) < 2 {
+		// invalid selector
+		return
+	}
+	k, v := splits[0], splits[1]
+	ifce, ok := jsonData[k]
+	if !ok {
+		return // the key doesn't exist
+	}
+	if m, ok := ifce.(map[string]interface{}); ok {
+		deleteJSONKey(v, m)
+	}
+}
+
 // UnmarshalJSON sets the fields of the handler.
 func (h *JSONHandler) UnmarshalJSON(data []byte) bool {
 	raw := make(map[string]interface{})
@@ -75,37 +117,38 @@ func (h *JSONHandler) UnmarshalJSON(data []byte) bool {
 		return false
 	}
 
-	checkEachUntilFound(supportedTimeFields, func(field string) bool {
-		time, ok := tryParseTime(raw[field])
+	if h.Opts == nil {
+		h.Opts = DefaultOptions
+	}
+
+	searchJSON(raw, h.Opts.TimeFields, func(field string, value interface{}) bool {
+		var ok bool
+		h.Time, ok = tryParseTime(value)
 		if ok {
-			h.Time = time
-			delete(raw, field)
+			deleteJSONKey(field, raw)
 		}
 		return ok
 	})
 
-	checkEachUntilFound(supportedMessageFields, func(field string) bool {
-		msg, ok := raw[field].(string)
+	searchJSON(raw, h.Opts.MessageFields, func(field string, value interface{}) bool {
+		var ok bool
+		h.Message, ok = value.(string)
 		if ok {
-			h.Message = msg
-			delete(raw, field)
+			deleteJSONKey(field, raw)
 		}
 		return ok
 	})
 
-	checkEachUntilFound(supportedLevelFields, func(field string) bool {
-		lvl, ok := raw[field]
-		if !ok {
-			return false
-		}
-		if strLvl, ok := lvl.(string); ok {
+	searchJSON(raw, h.Opts.LevelFields, func(field string, value interface{}) bool {
+		if strLvl, ok := value.(string); ok {
 			h.Level = strLvl
-		} else if flLvl, ok := lvl.(float64); ok {
+			deleteJSONKey(field, raw)
+		} else if flLvl, ok := value.(float64); ok {
 			h.Level = convertBunyanLogLevel(flLvl)
+			deleteJSONKey(field, raw)
 		} else {
 			h.Level = "???"
 		}
-		delete(raw, field)
 		return true
 	})