diff --git a/cmd/utils.go b/cmd/utils.go index d1d813fc87..331a65e580 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -130,6 +130,8 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) { switch format { + case yqlib.LuaInputFormat: + return yqlib.NewLuaDecoder(yqlib.ConfiguredLuaPreferences), nil case yqlib.XMLInputFormat: return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil case yqlib.PropertiesInputFormat: diff --git a/go.mod b/go.mod index ef06c29b61..cd5a5ac219 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/yuin/gopher-lua v1.1.0 golang.org/x/net v0.15.0 golang.org/x/text v0.13.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 diff --git a/go.sum b/go.sum index 825e41345a..061c4f2c36 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= +github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go index af5d3ecf01..284b4316f4 100644 --- a/pkg/yqlib/decoder.go +++ b/pkg/yqlib/decoder.go @@ -18,6 +18,7 @@ const ( TSVObjectInputFormat TomlInputFormat UriInputFormat + LuaInputFormat ) type Decoder interface { @@ -41,6 +42,8 @@ func InputFormatFromString(format string) (InputFormat, error) { return TSVObjectInputFormat, nil case "toml": return TomlInputFormat, nil + case "lua", "l": + return LuaInputFormat, nil default: return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format) } diff --git a/pkg/yqlib/decoder_lua.go b/pkg/yqlib/decoder_lua.go new file mode 100644 index 0000000000..5db879f4ad --- /dev/null +++ b/pkg/yqlib/decoder_lua.go @@ -0,0 +1,167 @@ +package yqlib + +import ( + "fmt" + "io" + "math" + + lua "github.com/yuin/gopher-lua" + yaml "gopkg.in/yaml.v3" +) + +type luaDecoder struct { + reader io.Reader + finished bool + prefs LuaPreferences +} + +func NewLuaDecoder(prefs LuaPreferences) Decoder { + return &luaDecoder{ + prefs: prefs, + } +} + +func (dec *luaDecoder) Init(reader io.Reader) error { + dec.reader = reader + return nil +} + +func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *yaml.Node { + switch lv.Type() { + case lua.LTNil: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!null", + Value: "", + } + case lua.LTBool: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!bool", + Value: lv.String(), + } + case lua.LTNumber: + n := float64(lua.LVAsNumber(lv)) + // various special case floats + if math.IsNaN(n) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: ".nan", + } + } + if math.IsInf(n, 1) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: ".inf", + } + } + if math.IsInf(n, -1) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: "-.inf", + } + } + + // does it look like an integer? + if n == float64(int(n)) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: lv.String(), + } + } + + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: lv.String(), + } + case lua.LTString: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: lv.String(), + } + case lua.LTFunction: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "tag:lua.org,2006,function", + Value: lv.String(), + } + case lua.LTTable: + // Simultaneously create a sequence and a map, pick which one to return + // based on whether all keys were consecutive integers + i := 1 + yaml_sequence := &yaml.Node{ + Kind: yaml.SequenceNode, + Tag: "!!seq", + } + yaml_map := &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + } + t := lv.(*lua.LTable) + k, v := ls.Next(t, lua.LNil) + for k != lua.LNil { + if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i { + i++ + } else { + i = 0 + } + yaml_map.Content = append(yaml_map.Content, dec.convertToYamlNode(ls, k)) + yv := dec.convertToYamlNode(ls, v) + yaml_map.Content = append(yaml_map.Content, yv) + if i != 0 { + yaml_sequence.Content = append(yaml_sequence.Content, yv) + } + k, v = ls.Next(t, k) + } + if i != 0 { + return yaml_sequence + } + return yaml_map + default: + return &yaml.Node{ + Kind: yaml.ScalarNode, + LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()), + Tag: "!!null", + Value: lv.String(), + } + } +} + +func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *yaml.Node { + if ls.GetTop() == 0 { + // no items were explicitly returned, encode the globals table instead + return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex)) + } + return dec.convertToYamlNode(ls, ls.Get(1)) +} + +func (dec *luaDecoder) Decode() (*CandidateNode, error) { + if dec.finished { + return nil, io.EOF + } + ls := lua.NewState(lua.Options{SkipOpenLibs: true}) + defer ls.Close() + fn, err := ls.Load(dec.reader, "@input") + if err != nil { + return nil, err + } + ls.Push(fn) + err = ls.PCall(0, lua.MultRet, nil) + if err != nil { + return nil, err + } + firstNode := dec.decideTopLevelNode(ls) + dec.finished = true + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{firstNode}, + }, + }, nil +} diff --git a/pkg/yqlib/doc/usage/lua.md b/pkg/yqlib/doc/usage/lua.md index 61458be288..d6f067e48c 100644 --- a/pkg/yqlib/doc/usage/lua.md +++ b/pkg/yqlib/doc/usage/lua.md @@ -1,5 +1,33 @@ -## Basic example +## Basic input example +Given a sample.lua file of: +```lua +return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; + +``` +then +```bash +yq -oy '.' sample.lua +``` +will output +```yaml +country: Australia +cities: + - Sydney + - Melbourne + - Brisbane + - Perth +``` + +## Basic output example Given a sample.yml file of: ```yaml --- diff --git a/pkg/yqlib/lua_test.go b/pkg/yqlib/lua_test.go index 3f8133d659..4089308f74 100644 --- a/pkg/yqlib/lua_test.go +++ b/pkg/yqlib/lua_test.go @@ -10,7 +10,27 @@ import ( var luaScenarios = []formatScenario{ { - description: "Basic example", + description: "Basic input example", + input: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + expected: `country: Australia +cities: + - Sydney + - Melbourne + - Brisbane + - Perth +`, + }, + { + description: "Basic output example", scenarioType: "encode", input: `--- country: Australia # this place @@ -28,6 +48,31 @@ cities: "Perth", }; }; +`, + }, + { + description: "Basic roundtrip", + skipDoc: true, + scenarioType: "roundtrip", + input: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + expected: `return { + ["country"] = "Australia"; + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; `, }, { @@ -168,8 +213,12 @@ numbers: func testLuaScenario(t *testing.T, s formatScenario) { switch s.scenarioType { + case "", "decode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description) case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) + case "roundtrip": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) case "unquoted-encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ DocPrefix: "return ", @@ -196,13 +245,39 @@ func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) { return } switch s.scenarioType { + case "", "decode": + documentLuaDecodeScenario(w, s) case "encode", "unquoted-encode", "globals-encode": documentLuaEncodeScenario(w, s) + case "roundtrip": + documentLuaRoundTripScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } +func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.lua file of:\n") + writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + expression := s.expression + if expression == "" { + expression = "." + } + writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression)) + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)))) +} + func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) @@ -245,6 +320,24 @@ func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs)))) } +func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.lua file of:\n") + writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + writeOrPanic(w, "```bash\nyq '.' sample.lua\n```\n") + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)))) +} + func TestLuaScenarios(t *testing.T) { for _, tt := range luaScenarios { testLuaScenario(t, tt) diff --git a/project-words.txt b/project-words.txt index cbbad84181..cbea0767ee 100644 --- a/project-words.txt +++ b/project-words.txt @@ -126,6 +126,7 @@ Lexer libdistro lindex linecomment +LVAs magiconair mapvalues Mier @@ -135,6 +136,7 @@ minishift mipsle mitchellh mktemp +Mult multidoc multimaint myenv @@ -244,4 +246,5 @@ xmld xyzzy yamld yqlib +yuin zabbix