diff --git a/.gitignore b/.gitignore index 79e0815..c4a29a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -gron +/gron *.tgz *.zip *.swp diff --git a/internal/gron/gron.go b/internal/gron/gron.go new file mode 100644 index 0000000..5383d2a --- /dev/null +++ b/internal/gron/gron.go @@ -0,0 +1,168 @@ +package gron + +import ( + "bufio" + "bytes" + "fmt" + "io" + "sort" + + "github.com/fatih/color" +) + +// Exit codes +const ( + exitOK = iota + exitOpenFile + exitReadInput + exitFormStatements + exitFetchURL + exitParseStatements + exitJSONEncode +) + +// Option bitfields +const ( + optMonochrome = 1 << iota + optNoSort + optJSON +) + +// Output colors +var ( + strColor = color.New(color.FgYellow) + braceColor = color.New(color.FgMagenta) + bareColor = color.New(color.FgBlue, color.Bold) + numColor = color.New(color.FgRed) + boolColor = color.New(color.FgCyan) +) + +// Gron is the default action. Given JSON as the input it returns a list +// of assignment statements. Possible options are optNoSort and optMonochrome +func Gron(r io.Reader, w io.Writer, opts int) (int, error) { + var err error + + var conv statementconv + if opts&optMonochrome > 0 { + conv = statementToString + } else { + conv = statementToColorString + } + + ss, err := statementsFromJSON(r, statement{{"json", typBare}}) + if err != nil { + goto out + } + + // Go's maps do not have well-defined ordering, but we want a consistent + // output for a given input, so we must sort the statements + if opts&optNoSort == 0 { + sort.Sort(ss) + } + + for _, s := range ss { + if opts&optJSON > 0 { + s, err = s.jsonify() + if err != nil { + goto out + } + } + fmt.Fprintln(w, conv(s)) + } + +out: + if err != nil { + return exitFormStatements, fmt.Errorf("failed to form statements: %s", err) + } + return exitOK, nil +} + +// GronStream is like the gron action, but it treats the input as one +// JSON object per line. There's a bit of code duplication from the +// gron action, but it'd be fairly messy to combine the two actions +func GronStream(r io.Reader, w io.Writer, opts int) (int, error) { + var err error + errstr := "failed to form statements" + var i int + var sc *bufio.Scanner + var buf []byte + + var conv func(s statement) string + if opts&optMonochrome > 0 { + conv = statementToString + } else { + conv = statementToColorString + } + + // Helper function to make the prefix statements for each line + makePrefix := func(index int) statement { + return statement{ + {"json", typBare}, + {"[", typLBrace}, + {fmt.Sprintf("%d", index), typNumericKey}, + {"]", typRBrace}, + } + } + + // The first line of output needs to establish that the top-level + // thing is actually an array... + top := statement{ + {"json", typBare}, + {"=", typEquals}, + {"[]", typEmptyArray}, + {";", typSemi}, + } + + if opts&optJSON > 0 { + top, err = top.jsonify() + if err != nil { + goto out + } + } + + fmt.Fprintln(w, conv(top)) + + // Read the input line by line + sc = bufio.NewScanner(r) + buf = make([]byte, 0, 64*1024) + sc.Buffer(buf, 1024*1024) + i = 0 + for sc.Scan() { + + line := bytes.NewBuffer(sc.Bytes()) + + var ss statements + ss, err = statementsFromJSON(line, makePrefix(i)) + i++ + if err != nil { + goto out + } + + // Go's maps do not have well-defined ordering, but we want a consistent + // output for a given input, so we must sort the statements + if opts&optNoSort == 0 { + sort.Sort(ss) + } + + for _, s := range ss { + if opts&optJSON > 0 { + s, err = s.jsonify() + if err != nil { + goto out + } + + } + fmt.Fprintln(w, conv(s)) + } + } + if err = sc.Err(); err != nil { + errstr = "error reading multiline input: %s" + } + +out: + if err != nil { + return exitFormStatements, fmt.Errorf(errstr+": %s", err) + } + return exitOK, nil + +} diff --git a/main_test.go b/internal/gron/gron_test.go similarity index 94% rename from main_test.go rename to internal/gron/gron_test.go index 94b57cd..ecb1257 100644 --- a/main_test.go +++ b/internal/gron/gron_test.go @@ -1,4 +1,4 @@ -package main +package gron import ( "bytes" @@ -32,7 +32,7 @@ func TestGron(t *testing.T) { } out := &bytes.Buffer{} - code, err := gron(in, out, optMonochrome) + code, err := Gron(in, out, optMonochrome) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -71,7 +71,7 @@ func TestGronStream(t *testing.T) { } out := &bytes.Buffer{} - code, err := gronStream(in, out, optMonochrome) + code, err := GronStream(in, out, optMonochrome) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -109,7 +109,7 @@ func TestLargeGronStream(t *testing.T) { } out := &bytes.Buffer{} - code, err := gronStream(in, out, optMonochrome) + code, err := GronStream(in, out, optMonochrome) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -158,7 +158,7 @@ func TestUngron(t *testing.T) { } out := &bytes.Buffer{} - code, err := ungron(in, out, optMonochrome) + code, err := Ungron(in, out, optMonochrome) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -205,7 +205,7 @@ func TestGronJ(t *testing.T) { } out := &bytes.Buffer{} - code, err := gron(in, out, optMonochrome|optJSON) + code, err := Gron(in, out, optMonochrome|optJSON) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -244,7 +244,7 @@ func TestGronStreamJ(t *testing.T) { } out := &bytes.Buffer{} - code, err := gronStream(in, out, optMonochrome|optJSON) + code, err := GronStream(in, out, optMonochrome|optJSON) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -291,7 +291,7 @@ func TestUngronJ(t *testing.T) { } out := &bytes.Buffer{} - code, err := ungron(in, out, optMonochrome|optJSON) + code, err := Ungron(in, out, optMonochrome|optJSON) if code != exitOK { t.Errorf("want exitOK; have %d", code) @@ -328,7 +328,7 @@ func BenchmarkBigJSON(b *testing.B) { b.Fatalf("failed to rewind input: %s", err) } - _, err := gron(in, out, optMonochrome|optNoSort) + _, err := Gron(in, out, optMonochrome|optNoSort) if err != nil { b.Fatalf("failed to gron: %s", err) } diff --git a/identifier.go b/internal/gron/identifier.go similarity index 99% rename from identifier.go rename to internal/gron/identifier.go index 14f6eaf..e6346a1 100644 --- a/identifier.go +++ b/internal/gron/identifier.go @@ -1,4 +1,4 @@ -package main +package gron import "unicode" diff --git a/identifier_test.go b/internal/gron/identifier_test.go similarity index 99% rename from identifier_test.go rename to internal/gron/identifier_test.go index 82df524..504c12a 100644 --- a/identifier_test.go +++ b/internal/gron/identifier_test.go @@ -1,4 +1,4 @@ -package main +package gron import "testing" diff --git a/statements.go b/internal/gron/statements.go similarity index 99% rename from statements.go rename to internal/gron/statements.go index ed1b469..c1a7a92 100644 --- a/statements.go +++ b/internal/gron/statements.go @@ -1,4 +1,4 @@ -package main +package gron import ( "encoding/json" diff --git a/statements_test.go b/internal/gron/statements_test.go similarity index 99% rename from statements_test.go rename to internal/gron/statements_test.go index 12137a3..ff14d47 100644 --- a/statements_test.go +++ b/internal/gron/statements_test.go @@ -1,4 +1,4 @@ -package main +package gron import ( "bytes" diff --git a/testdata/big.json b/internal/gron/testdata/big.json similarity index 100% rename from testdata/big.json rename to internal/gron/testdata/big.json diff --git a/testdata/github.gron b/internal/gron/testdata/github.gron similarity index 100% rename from testdata/github.gron rename to internal/gron/testdata/github.gron diff --git a/testdata/github.jgron b/internal/gron/testdata/github.jgron similarity index 100% rename from testdata/github.jgron rename to internal/gron/testdata/github.jgron diff --git a/testdata/github.json b/internal/gron/testdata/github.json similarity index 100% rename from testdata/github.json rename to internal/gron/testdata/github.json diff --git a/testdata/grep-separators.gron b/internal/gron/testdata/grep-separators.gron similarity index 100% rename from testdata/grep-separators.gron rename to internal/gron/testdata/grep-separators.gron diff --git a/testdata/grep-separators.json b/internal/gron/testdata/grep-separators.json similarity index 100% rename from testdata/grep-separators.json rename to internal/gron/testdata/grep-separators.json diff --git a/testdata/invalid-type-mismatch.gron b/internal/gron/testdata/invalid-type-mismatch.gron similarity index 100% rename from testdata/invalid-type-mismatch.gron rename to internal/gron/testdata/invalid-type-mismatch.gron diff --git a/testdata/invalid-value.gron b/internal/gron/testdata/invalid-value.gron similarity index 100% rename from testdata/invalid-value.gron rename to internal/gron/testdata/invalid-value.gron diff --git a/testdata/large-line.gron b/internal/gron/testdata/large-line.gron similarity index 100% rename from testdata/large-line.gron rename to internal/gron/testdata/large-line.gron diff --git a/testdata/large-line.json b/internal/gron/testdata/large-line.json similarity index 100% rename from testdata/large-line.json rename to internal/gron/testdata/large-line.json diff --git a/testdata/long-stream.gron b/internal/gron/testdata/long-stream.gron similarity index 100% rename from testdata/long-stream.gron rename to internal/gron/testdata/long-stream.gron diff --git a/testdata/long-stream.json b/internal/gron/testdata/long-stream.json similarity index 100% rename from testdata/long-stream.json rename to internal/gron/testdata/long-stream.json diff --git a/testdata/one.gron b/internal/gron/testdata/one.gron similarity index 100% rename from testdata/one.gron rename to internal/gron/testdata/one.gron diff --git a/testdata/one.jgron b/internal/gron/testdata/one.jgron similarity index 100% rename from testdata/one.jgron rename to internal/gron/testdata/one.jgron diff --git a/testdata/one.json b/internal/gron/testdata/one.json similarity index 100% rename from testdata/one.json rename to internal/gron/testdata/one.json diff --git a/testdata/scalar-stream.gron b/internal/gron/testdata/scalar-stream.gron similarity index 100% rename from testdata/scalar-stream.gron rename to internal/gron/testdata/scalar-stream.gron diff --git a/testdata/scalar-stream.jgron b/internal/gron/testdata/scalar-stream.jgron similarity index 100% rename from testdata/scalar-stream.jgron rename to internal/gron/testdata/scalar-stream.jgron diff --git a/testdata/scalar-stream.json b/internal/gron/testdata/scalar-stream.json similarity index 100% rename from testdata/scalar-stream.json rename to internal/gron/testdata/scalar-stream.json diff --git a/testdata/stream.gron b/internal/gron/testdata/stream.gron similarity index 100% rename from testdata/stream.gron rename to internal/gron/testdata/stream.gron diff --git a/testdata/stream.jgron b/internal/gron/testdata/stream.jgron similarity index 100% rename from testdata/stream.jgron rename to internal/gron/testdata/stream.jgron diff --git a/testdata/stream.json b/internal/gron/testdata/stream.json similarity index 100% rename from testdata/stream.json rename to internal/gron/testdata/stream.json diff --git a/testdata/three.gron b/internal/gron/testdata/three.gron similarity index 100% rename from testdata/three.gron rename to internal/gron/testdata/three.gron diff --git a/testdata/three.jgron b/internal/gron/testdata/three.jgron similarity index 100% rename from testdata/three.jgron rename to internal/gron/testdata/three.jgron diff --git a/testdata/three.json b/internal/gron/testdata/three.json similarity index 100% rename from testdata/three.json rename to internal/gron/testdata/three.json diff --git a/testdata/two-b.json b/internal/gron/testdata/two-b.json similarity index 100% rename from testdata/two-b.json rename to internal/gron/testdata/two-b.json diff --git a/testdata/two.gron b/internal/gron/testdata/two.gron similarity index 100% rename from testdata/two.gron rename to internal/gron/testdata/two.gron diff --git a/testdata/two.jgron b/internal/gron/testdata/two.jgron similarity index 100% rename from testdata/two.jgron rename to internal/gron/testdata/two.jgron diff --git a/testdata/two.json b/internal/gron/testdata/two.json similarity index 100% rename from testdata/two.json rename to internal/gron/testdata/two.json diff --git a/token.go b/internal/gron/token.go similarity index 99% rename from token.go rename to internal/gron/token.go index 0bc5865..a444d39 100644 --- a/token.go +++ b/internal/gron/token.go @@ -1,4 +1,4 @@ -package main +package gron import ( "bytes" diff --git a/token_test.go b/internal/gron/token_test.go similarity index 98% rename from token_test.go rename to internal/gron/token_test.go index 91e4bae..5c91de0 100644 --- a/token_test.go +++ b/internal/gron/token_test.go @@ -1,4 +1,4 @@ -package main +package gron import ( "encoding/json" diff --git a/ungron.go b/internal/gron/ungron.go similarity index 80% rename from ungron.go rename to internal/gron/ungron.go index 453ada6..c1c5bff 100644 --- a/ungron.go +++ b/internal/gron/ungron.go @@ -10,19 +10,122 @@ // String ::= '"' (UnescapedRune | ("\" (["\/bfnrt] | ('u' Hex))))* '"' // UnescapedRune ::= [^#x0-#x1f"\] -package main +package gron import ( "encoding/json" "fmt" + "io" "strconv" + "bufio" + "bytes" "strings" "unicode" "unicode/utf8" "github.com/pkg/errors" + "github.com/nwidger/jsoncolor" ) +// Ungron is the reverse of gron. Given assignment statements as input, +// it returns JSON. The only option is optMonochrome +func Ungron(r io.Reader, w io.Writer, opts int) (int, error) { + scanner := bufio.NewScanner(r) + var maker statementmaker + + // Allow larger internal buffer of the scanner (min: 64KiB ~ max: 1MiB) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + + if opts&optJSON > 0 { + maker = statementFromJSONSpec + } else { + maker = statementFromStringMaker + } + + // Make a list of statements from the input + var ss statements + for scanner.Scan() { + s, err := maker(scanner.Text()) + if err != nil { + return exitParseStatements, err + } + ss.add(s) + } + if err := scanner.Err(); err != nil { + return exitReadInput, fmt.Errorf("failed to read input statements") + } + + // turn the statements into a single merged interface{} type + merged, err := ss.toInterface() + if err != nil { + return exitParseStatements, err + } + + // If there's only one top level key and it's "json", make that the top level thing + mergedMap, ok := merged.(map[string]interface{}) + if ok { + if len(mergedMap) == 1 { + if _, exists := mergedMap["json"]; exists { + merged = mergedMap["json"] + } + } + } + + // Marshal the output into JSON to display to the user + out := &bytes.Buffer{} + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + enc.SetEscapeHTML(false) + err = enc.Encode(merged) + if err != nil { + return exitJSONEncode, errors.Wrap(err, "failed to convert statements to JSON") + } + j := out.Bytes() + + // If the output isn't monochrome, add color to the JSON + if opts&optMonochrome == 0 { + c, err := colorizeJSON(j) + + // If we failed to colorize the JSON for whatever reason, + // we'll just fall back to monochrome output, otherwise + // replace the monochrome JSON with glorious technicolor + if err == nil { + j = c + } + } + + // For whatever reason, the monochrome version of the JSON + // has a trailing newline character, but the colorized version + // does not. Strip the whitespace so that neither has the newline + // character on the end, and then we'll add a newline in the + // Fprintf below + j = bytes.TrimSpace(j) + + fmt.Fprintf(w, "%s\n", j) + + return exitOK, nil +} + +func colorizeJSON(src []byte) ([]byte, error) { + out := &bytes.Buffer{} + f := jsoncolor.NewFormatter() + + f.StringColor = strColor + f.ObjectColor = braceColor + f.ArrayColor = braceColor + f.FieldColor = bareColor + f.NumberColor = numColor + f.TrueColor = boolColor + f.FalseColor = boolColor + f.NullColor = boolColor + + err := f.Format(out, src) + if err != nil { + return out.Bytes(), err + } + return out.Bytes(), nil +} + // errRecoverable is an error type to represent errors that // can be recovered from; e.g. an empty line in the input type errRecoverable struct { diff --git a/ungron_test.go b/internal/gron/ungron_test.go similarity index 99% rename from ungron_test.go rename to internal/gron/ungron_test.go index f9f3fb4..63086b5 100644 --- a/ungron_test.go +++ b/internal/gron/ungron_test.go @@ -1,4 +1,4 @@ -package main +package gron import ( "reflect" diff --git a/main.go b/main.go index 840e36a..8ccdaa5 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,15 @@ package main import ( - "bufio" - "bytes" - "encoding/json" "flag" "fmt" "io" "os" - "sort" "github.com/fatih/color" "github.com/mattn/go-colorable" - "github.com/nwidger/jsoncolor" - "github.com/pkg/errors" + + "github.com/tomnomnom/gron/internal/gron" ) // Exit codes @@ -156,11 +152,11 @@ func main() { } // Pick the appropriate action: gron, ungron or gronStream - var a actionFn = gron + var a actionFn = gron.Gron if ungronFlag { - a = ungron + a = gron.Ungron } else if streamFlag { - a = gronStream + a = gron.GronStream } exitCode, err := a(rawInput, colorable.NewColorableStdout(), opts) @@ -176,235 +172,6 @@ func main() { // code and any error that occurred type actionFn func(io.Reader, io.Writer, int) (int, error) -// gron is the default action. Given JSON as the input it returns a list -// of assignment statements. Possible options are optNoSort and optMonochrome -func gron(r io.Reader, w io.Writer, opts int) (int, error) { - var err error - - var conv statementconv - if opts&optMonochrome > 0 { - conv = statementToString - } else { - conv = statementToColorString - } - - ss, err := statementsFromJSON(r, statement{{"json", typBare}}) - if err != nil { - goto out - } - - // Go's maps do not have well-defined ordering, but we want a consistent - // output for a given input, so we must sort the statements - if opts&optNoSort == 0 { - sort.Sort(ss) - } - - for _, s := range ss { - if opts&optJSON > 0 { - s, err = s.jsonify() - if err != nil { - goto out - } - } - fmt.Fprintln(w, conv(s)) - } - -out: - if err != nil { - return exitFormStatements, fmt.Errorf("failed to form statements: %s", err) - } - return exitOK, nil -} - -// gronStream is like the gron action, but it treats the input as one -// JSON object per line. There's a bit of code duplication from the -// gron action, but it'd be fairly messy to combine the two actions -func gronStream(r io.Reader, w io.Writer, opts int) (int, error) { - var err error - errstr := "failed to form statements" - var i int - var sc *bufio.Scanner - var buf []byte - - var conv func(s statement) string - if opts&optMonochrome > 0 { - conv = statementToString - } else { - conv = statementToColorString - } - - // Helper function to make the prefix statements for each line - makePrefix := func(index int) statement { - return statement{ - {"json", typBare}, - {"[", typLBrace}, - {fmt.Sprintf("%d", index), typNumericKey}, - {"]", typRBrace}, - } - } - - // The first line of output needs to establish that the top-level - // thing is actually an array... - top := statement{ - {"json", typBare}, - {"=", typEquals}, - {"[]", typEmptyArray}, - {";", typSemi}, - } - - if opts&optJSON > 0 { - top, err = top.jsonify() - if err != nil { - goto out - } - } - - fmt.Fprintln(w, conv(top)) - - // Read the input line by line - sc = bufio.NewScanner(r) - buf = make([]byte, 0, 64*1024) - sc.Buffer(buf, 1024*1024) - i = 0 - for sc.Scan() { - - line := bytes.NewBuffer(sc.Bytes()) - - var ss statements - ss, err = statementsFromJSON(line, makePrefix(i)) - i++ - if err != nil { - goto out - } - - // Go's maps do not have well-defined ordering, but we want a consistent - // output for a given input, so we must sort the statements - if opts&optNoSort == 0 { - sort.Sort(ss) - } - - for _, s := range ss { - if opts&optJSON > 0 { - s, err = s.jsonify() - if err != nil { - goto out - } - - } - fmt.Fprintln(w, conv(s)) - } - } - if err = sc.Err(); err != nil { - errstr = "error reading multiline input: %s" - } - -out: - if err != nil { - return exitFormStatements, fmt.Errorf(errstr+": %s", err) - } - return exitOK, nil - -} - -// ungron is the reverse of gron. Given assignment statements as input, -// it returns JSON. The only option is optMonochrome -func ungron(r io.Reader, w io.Writer, opts int) (int, error) { - scanner := bufio.NewScanner(r) - var maker statementmaker - - // Allow larger internal buffer of the scanner (min: 64KiB ~ max: 1MiB) - scanner.Buffer(make([]byte, 64*1024), 1024*1024) - - if opts&optJSON > 0 { - maker = statementFromJSONSpec - } else { - maker = statementFromStringMaker - } - - // Make a list of statements from the input - var ss statements - for scanner.Scan() { - s, err := maker(scanner.Text()) - if err != nil { - return exitParseStatements, err - } - ss.add(s) - } - if err := scanner.Err(); err != nil { - return exitReadInput, fmt.Errorf("failed to read input statements") - } - - // turn the statements into a single merged interface{} type - merged, err := ss.toInterface() - if err != nil { - return exitParseStatements, err - } - - // If there's only one top level key and it's "json", make that the top level thing - mergedMap, ok := merged.(map[string]interface{}) - if ok { - if len(mergedMap) == 1 { - if _, exists := mergedMap["json"]; exists { - merged = mergedMap["json"] - } - } - } - - // Marshal the output into JSON to display to the user - out := &bytes.Buffer{} - enc := json.NewEncoder(out) - enc.SetIndent("", " ") - enc.SetEscapeHTML(false) - err = enc.Encode(merged) - if err != nil { - return exitJSONEncode, errors.Wrap(err, "failed to convert statements to JSON") - } - j := out.Bytes() - - // If the output isn't monochrome, add color to the JSON - if opts&optMonochrome == 0 { - c, err := colorizeJSON(j) - - // If we failed to colorize the JSON for whatever reason, - // we'll just fall back to monochrome output, otherwise - // replace the monochrome JSON with glorious technicolor - if err == nil { - j = c - } - } - - // For whatever reason, the monochrome version of the JSON - // has a trailing newline character, but the colorized version - // does not. Strip the whitespace so that neither has the newline - // character on the end, and then we'll add a newline in the - // Fprintf below - j = bytes.TrimSpace(j) - - fmt.Fprintf(w, "%s\n", j) - - return exitOK, nil -} - -func colorizeJSON(src []byte) ([]byte, error) { - out := &bytes.Buffer{} - f := jsoncolor.NewFormatter() - - f.StringColor = strColor - f.ObjectColor = braceColor - f.ArrayColor = braceColor - f.FieldColor = bareColor - f.NumberColor = numColor - f.TrueColor = boolColor - f.FalseColor = boolColor - f.NullColor = boolColor - - err := f.Format(out, src) - if err != nil { - return out.Bytes(), err - } - return out.Bytes(), nil -} - func fatal(code int, err error) { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(code)