Add ByteString API

This is a breaking change that adds a `AddByteString` to `zapcore.ObjectEncoder`, `AppendByteString` to `zapcore.PrimitiveArrayEncoder`.
This API optimizes logging UTF-8 encoded []byte data.

Fixes uber-go#324.

array.go

@@ -43,6 +43,12 @@ func Bools(key string, bs []bool) zapcore.Field {
 	return Array(key, bools(bs))
 }
 
+// ByteStrings constructs a field that carries a slice of []byte
+// that assumed to be UTF-8 encoded.
+func ByteStrings(key string, bss [][]byte) zapcore.Field {
+	return Array(key, byteStringsArray(bss))
+}
+
 // Complex128s constructs a field that carries a slice of complex numbers.
 func Complex128s(key string, nums []complex128) zapcore.Field {
 	return Array(key, complex128s(nums))
@@ -147,6 +153,15 @@ func (bs bools) MarshalLogArray(arr zapcore.ArrayEncoder) error {
 	return nil
 }
 
+type byteStringsArray [][]byte
+
+func (bss byteStringsArray) MarshalLogArray(arr zapcore.ArrayEncoder) error {
+	for i := range bss {
+		arr.AppendByteString(bss[i])
+	}
+	return nil
+}
+
 type complex128s []complex128
 
 func (nums complex128s) MarshalLogArray(arr zapcore.ArrayEncoder) error {

array_test.go

@@ -25,11 +25,10 @@ import (
 	"testing"
 	"time"
 
-	"go.uber.org/zap/zapcore"
-
 	richErrors "github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zapcore"
 )
 
 func BenchmarkBoolsArrayMarshaler(b *testing.B) {
@@ -59,6 +58,7 @@ func TestArrayWrappers(t *testing.T) {
 		expected []interface{}
 	}{
 		{"empty bools", Bools("", []bool{}), []interface{}(nil)},
+		{"empty byte strings", ByteStrings("", [][]byte{}), []interface{}(nil)},
 		{"empty complex128s", Complex128s("", []complex128{}), []interface{}(nil)},
 		{"empty complex64s", Complex64s("", []complex64{}), []interface{}(nil)},
 		{"empty durations", Durations("", []time.Duration{}), []interface{}(nil)},
@@ -79,6 +79,7 @@ func TestArrayWrappers(t *testing.T) {
 		{"empty uintptrs", Uintptrs("", []uintptr{}), []interface{}(nil)},
 		{"empty errors", Errors("", []error{}), []interface{}(nil)},
 		{"bools", Bools("", []bool{true, false}), []interface{}{true, false}},
+		{"byte strings", ByteStrings("", [][]byte{{1, 2}, {3, 4}}), []interface{}{[]byte{1, 2}, []byte{3, 4}}},
 		{"complex128s", Complex128s("", []complex128{1 + 2i, 3 + 4i}), []interface{}{1 + 2i, 3 + 4i}},
 		{"complex64s", Complex64s("", []complex64{1 + 2i, 3 + 4i}), []interface{}{complex64(1 + 2i), complex64(3 + 4i)}},
 		{"durations", Durations("", []time.Duration{1, 2}), []interface{}{time.Nanosecond, 2 * time.Nanosecond}},

field.go

@@ -50,6 +50,14 @@ func Bool(key string, val bool) zapcore.Field {
 	return zapcore.Field{Key: key, Type: zapcore.BoolType, Integer: ival}
 }
 
+// ByteString constructs a field that carries a []byte that assumed to be UTF-8 encoded.
+//
+// Saves on []byte to string cast copy and allocation, but costs smaller allocation to convert
+// the []byte to interface{}.
+func ByteString(key string, val []byte) zapcore.Field {
+	return zapcore.Field{Key: key, Type: zapcore.ByteStringType, Interface: val}
+}
+
 // Complex128 constructs a field that carries a complex number. Unlike most
 // numeric fields, this costs an allocation (to convert the complex128 to
 // interface{}).

field_test.go

@@ -74,6 +74,7 @@ func TestFieldConstructors(t *testing.T) {
 		{"Binary", zapcore.Field{Key: "k", Type: zapcore.BinaryType, Interface: []byte("ab12")}, Binary("k", []byte("ab12"))},
 		{"Bool", zapcore.Field{Key: "k", Type: zapcore.BoolType, Integer: 1}, Bool("k", true)},
 		{"Bool", zapcore.Field{Key: "k", Type: zapcore.BoolType, Integer: 1}, Bool("k", true)},
+		{"ByteString", zapcore.Field{Key: "k", Type: zapcore.ByteStringType, Interface: []byte("ab12")}, ByteString("k", []byte("ab12"))},
 		{"Complex128", zapcore.Field{Key: "k", Type: zapcore.Complex128Type, Interface: 1 + 2i}, Complex128("k", 1+2i)},
 		{"Complex64", zapcore.Field{Key: "k", Type: zapcore.Complex64Type, Interface: complex64(1 + 2i)}, Complex64("k", 1+2i)},
 		{"Duration", zapcore.Field{Key: "k", Type: zapcore.DurationType, Integer: 1}, Duration("k", 1)},

logger_bench_test.go

@@ -75,6 +75,13 @@ func BenchmarkBoolField(b *testing.B) {
 	})
 }
 
+func BenchmarkByteStringField(b *testing.B) {
+	val := []byte("bar")
+	withBenchedLogger(b, func(log *Logger) {
+		log.Info("ByteString.", ByteString("foo", val))
+	})
+}
+
 func BenchmarkFloat64Field(b *testing.B) {
 	withBenchedLogger(b, func(log *Logger) {
 		log.Info("Floating point.", Float64("foo", 3.14))

zapcore/encoder.go

@@ -216,11 +216,17 @@ type EncoderConfig struct {
 // aren't safe for concurrent use (though typical use shouldn't require locks).
 type ObjectEncoder interface {
 	// Logging-specific marshalers.
+
 	AddArray(key string, marshaler ArrayMarshaler) error
 	AddObject(key string, marshaler ObjectMarshaler) error
 
 	// Built-in types.
+
+	// AddBinary adds raw blob of binary data.
 	AddBinary(key string, value []byte)
+	// AddByteString adds bytes as UTF-8 string.
+	// No-alloc equivalent of AddString(string(value)) for []byte values.
+	AddByteString(key string, value []byte)
 	AddBool(key string, value bool)
 	AddComplex128(key string, value complex128)
 	AddComplex64(key string, value complex64)
@@ -277,6 +283,7 @@ type ArrayEncoder interface {
 type PrimitiveArrayEncoder interface {
 	// Built-in types.
 	AppendBool(bool)
+	AppendByteString([]byte)
 	AppendComplex128(complex128)
 	AppendComplex64(complex64)
 	AppendFloat64(float64)

zapcore/field.go

@@ -41,6 +41,8 @@ const (
 	BinaryType
 	// BoolType indicates that the field carries a bool.
 	BoolType
+	// ByteStringType indicates that the field carries a []byte that assumed to be UTF-8 encoded.
+	ByteStringType
 	// Complex128Type indicates that the field carries a complex128.
 	Complex128Type
 	// Complex64Type indicates that the field carries a complex128.
@@ -112,6 +114,8 @@ func (f Field) AddTo(enc ObjectEncoder) {
 		enc.AddBinary(f.Key, f.Interface.([]byte))
 	case BoolType:
 		enc.AddBool(f.Key, f.Integer == 1)
+	case ByteStringType:
+		enc.AddByteString(f.Key, f.Interface.([]byte))
 	case Complex128Type:
 		enc.AddComplex128(f.Key, f.Interface.(complex128))
 	case Complex64Type:

zapcore/field_test.go

@@ -107,6 +107,7 @@ func TestFields(t *testing.T) {
 		{t: ObjectMarshalerType, iface: users(2), want: map[string]interface{}{"users": 2}},
 		{t: BinaryType, iface: []byte("foo"), want: []byte("foo")},
 		{t: BoolType, i: 0, want: false},
+		{t: ByteStringType, iface: []byte("foo"), want: []byte("foo")},
 		{t: Complex128Type, iface: 1 + 2i, want: 1 + 2i},
 		{t: Complex64Type, iface: complex64(1 + 2i), want: complex64(1 + 2i)},
 		{t: DurationType, i: 1000, want: time.Microsecond},

zapcore/json_encoder.go

@@ -94,6 +94,11 @@ func (enc *jsonEncoder) AddBinary(key string, val []byte) {
 	enc.AddString(key, base64.StdEncoding.EncodeToString(val))
 }
 
+func (enc *jsonEncoder) AddByteString(key string, val []byte) {
+	enc.addKey(key)
+	enc.AppendByteString(val)
+}
+
 func (enc *jsonEncoder) AddBool(key string, val bool) {
 	enc.addKey(key)
 	enc.AppendBool(val)
@@ -171,6 +176,13 @@ func (enc *jsonEncoder) AppendBool(val bool) {
 	enc.buf.AppendBool(val)
 }
 
+func (enc *jsonEncoder) AppendByteString(val []byte) {
+	enc.addElementSeparator()
+	enc.buf.AppendByte('"')
+	enc.safeAddByteString(val)
+	enc.buf.AppendByte('"')
+}
+
 func (enc *jsonEncoder) AppendComplex128(val complex128) {
 	enc.addElementSeparator()
 	// Cast to a platform-independent, fixed-size type.
@@ -379,40 +391,73 @@ func (enc *jsonEncoder) appendFloat(val float64, bitSize int) {
 // user from browser vulnerabilities or JSONP-related problems.
 func (enc *jsonEncoder) safeAddString(s string) {
 	for i := 0; i < len(s); {
-		if b := s[i]; b < utf8.RuneSelf {
+		if enc.tryAddRuneSelf(s[i]) {
 			i++
-			if 0x20 <= b && b != '\\' && b != '"' {
-				enc.buf.AppendByte(b)
-				continue
-			}
-			switch b {
-			case '\\', '"':
-				enc.buf.AppendByte('\\')
-				enc.buf.AppendByte(b)
-			case '\n':
-				enc.buf.AppendByte('\\')
-				enc.buf.AppendByte('n')
-			case '\r':
-				enc.buf.AppendByte('\\')
-				enc.buf.AppendByte('r')
-			case '\t':
-				enc.buf.AppendByte('\\')
-				enc.buf.AppendByte('t')
-			default:
-				// Encode bytes < 0x20, except for the escape sequences above.
-				enc.buf.AppendString(`\u00`)
-				enc.buf.AppendByte(_hex[b>>4])
-				enc.buf.AppendByte(_hex[b&0xF])
-			}
 			continue
 		}
-		c, size := utf8.DecodeRuneInString(s[i:])
-		if c == utf8.RuneError && size == 1 {
-			enc.buf.AppendString(`\ufffd`)
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if enc.tryAddRuneError(r, size) {
 			i++
 			continue
 		}
 		enc.buf.AppendString(s[i : i+size])
 		i += size
 	}
 }
+
+// safeAddByteString is no-alloc equivalent of safeAddString(string(s)) for s []byte.
+func (enc *jsonEncoder) safeAddByteString(s []byte) {
+	for i := 0; i < len(s); {
+		if enc.tryAddRuneSelf(s[i]) {
+			i++
+			continue
+		}
+		r, size := utf8.DecodeRune(s[i:])
+		if enc.tryAddRuneError(r, size) {
+			i++
+			continue
+		}
+		enc.buf.Write(s[i : i+size])
+		i += size
+	}
+}
+
+// tryAddRuneSelf appends b if it is valid UTF-8 character represented in a single byte.
+func (enc *jsonEncoder) tryAddRuneSelf(b byte) (ok bool) {
+	ok = b < utf8.RuneSelf
+	if !ok {
+		return
+	}
+	if 0x20 <= b && b != '\\' && b != '"' {
+		enc.buf.AppendByte(b)
+		return
+	}
+	switch b {
+	case '\\', '"':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte(b)
+	case '\n':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte('n')
+	case '\r':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte('r')
+	case '\t':
+		enc.buf.AppendByte('\\')
+		enc.buf.AppendByte('t')
+	default:
+		// Encode bytes < 0x20, except for the escape sequences above.
+		enc.buf.AppendString(`\u00`)
+		enc.buf.AppendByte(_hex[b>>4])
+		enc.buf.AppendByte(_hex[b&0xF])
+	}
+	return
+}
+
+func (enc *jsonEncoder) tryAddRuneError(r rune, size int) (ok bool) {
+	ok = r == utf8.RuneError && size == 1
+	if ok {
+		enc.buf.AppendString(`\ufffd`)
+	}
+	return
+}