From 5ef1c22b06a07abfd7729439f516f6a8fec537d8 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 15:51:58 +0300 Subject: [PATCH 01/12] init --- .github/workflows/unittest.yml | 6 ++++++ Makefile | 8 ++++++++ go.mod | 30 ++++++++++++++++++------------ go.sum | 29 +++++++++++++++++++++-------- schema.json | 27 +++++++++++++++++++++++++++ schemagen/main.go | 26 ++++++++++++++++++++++++++ spec.go | 5 +++++ 7 files changed, 111 insertions(+), 20 deletions(-) create mode 100644 schema.json create mode 100644 schemagen/main.go diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 28a876ee..341dbc75 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -19,5 +19,11 @@ jobs: go-version-file: go.mod - run: go mod download - run: go build ./... + - name: gen + if: github.event_name == 'pull_request' + run: make gen + - name: Fail if generation updated files + if: github.event_name == 'pull_request' + run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1) - name: Run tests run: go test -race ./... diff --git a/Makefile b/Makefile index fb63d98a..939b73ad 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,11 @@ test: .PHONY: lint lint: golangci-lint run + +.PHONY: gen-spec-schema +gen-spec-schema: + go run schemagen/main.go + +# All gen targets +.PHONY: gen +gen: gen-spec-schema diff --git a/go.mod b/go.mod index 426ab268..81ca72eb 100644 --- a/go.mod +++ b/go.mod @@ -5,24 +5,20 @@ go 1.21.1 require ( github.com/apache/arrow/go/v14 v14.0.0-20230929201650-00efb06dc0de github.com/bradleyjkemp/cupaloy/v2 v2.8.0 + github.com/cloudquery/codegen v0.3.10 github.com/cloudquery/plugin-sdk/v4 v4.16.1 github.com/goccy/go-json v0.10.2 github.com/google/uuid v1.3.1 + github.com/invopop/jsonschema v0.11.0 github.com/stretchr/testify v1.8.4 ) -require ( - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect -) - -// TODO: remove once all updates are merged -replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c - require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/thrift v0.19.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -30,6 +26,7 @@ require ( github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect @@ -37,17 +34,26 @@ require ( github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/zerolog v1.31.0 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/thoas/go-funk v0.9.3 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/mod v0.12.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.13.0 // indirect; indirect // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/grpc v1.58.2 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +// TODO: remove once all updates are merged +replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c + +// github.com/cloudquery/jsonschema @ cqmain +replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f diff --git a/go.sum b/go.sum index a3dee351..57c01f85 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,18 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk= github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c h1:z0ZtXvDjtohLaNHLGl25gyq7ULS0y64CayNvReJJCTc= github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c/go.mod h1:EkHPhLB+98ANnPojOy2sUhM0rzYbPEmrtuA9v8aZp/c= +github.com/cloudquery/codegen v0.3.10 h1:xd6g8UjxnMHzpY/TcM4r1YCcQ75tyc2Ehyu8d+yCpm4= +github.com/cloudquery/codegen v0.3.10/go.mod h1:ZqCxX9AGSMUnafE3gKmcT+RJCTgFg7izDxJwFSLGRcg= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8= +github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= github.com/cloudquery/plugin-sdk/v4 v4.16.1 h1:Ir2fkqsu5htnnI4wTGTmA6wp1LkixZmmbyxRSIsGoxM= github.com/cloudquery/plugin-sdk/v4 v4.16.1/go.mod h1:ujSFEUAp8BmozOee0ljjsPHQfddXJCUTAzCD6sVKsu8= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -29,6 +37,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= @@ -39,6 +48,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -70,18 +81,20 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -90,8 +103,8 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= diff --git a/schema.json b/schema.json new file mode 100644 index 00000000..6d4b06d9 --- /dev/null +++ b/schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/cloudquery/filetypes/v4/file-spec", + "$ref": "#/$defs/FileSpec", + "$defs": { + "FileSpec": { + "properties": { + "format": { + "type": "string" + }, + "format_spec": { + "oneOf": [ + true, + { + "type": "null" + } + ] + }, + "compression": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + } + } +} diff --git a/schemagen/main.go b/schemagen/main.go new file mode 100644 index 00000000..58d664c2 --- /dev/null +++ b/schemagen/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "log" + "path" + "runtime" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/cloudquery/filetypes/v4" +) + +func main() { + fmt.Println("Generating JSON schema for plugin spec") + jsonschema.GenerateIntoFile(new(filetypes.FileSpec), path.Join(currDir(), "..", "schema.json"), + jsonschema.WithAddGoComments("github.com/cloudquery/filetypes/v4", path.Join(currDir(), "..")), + ) +} + +func currDir() string { + _, filename, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Failed to get caller information") + } + return path.Dir(filename) +} diff --git a/spec.go b/spec.go index 01d720d7..ca43ef3a 100644 --- a/spec.go +++ b/spec.go @@ -8,6 +8,7 @@ import ( "github.com/cloudquery/filetypes/v4/csv" jsonFile "github.com/cloudquery/filetypes/v4/json" "github.com/cloudquery/filetypes/v4/parquet" + "github.com/invopop/jsonschema" ) type FormatType string @@ -34,6 +35,10 @@ type FileSpec struct { parquetSpec *parquet.Spec } +func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { + +} + func (s *FileSpec) SetDefaults() { switch s.Format { case FormatTypeCSV: From 43ba7d4c1ea6882c28a4634b07f45cbcc4239ab9 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 16:03:22 +0300 Subject: [PATCH 02/12] move to single interface impl --- client.go | 12 +----------- csv/read.go | 4 ++-- json/read.go | 4 ++-- parquet/read.go | 4 ++-- read.go | 27 +++------------------------ spec.go | 1 - types/types.go | 7 +++++++ 7 files changed, 17 insertions(+), 42 deletions(-) diff --git a/client.go b/client.go index 4abb9c34..c366a48f 100644 --- a/client.go +++ b/client.go @@ -10,10 +10,6 @@ import ( type Client struct { spec *FileSpec filetype types.FileType - - csv *csvFile.Client - json *jsonFile.Client - parquet *parquet.Client } var ( @@ -50,7 +46,6 @@ func NewClient(spec *FileSpec) (*Client, error) { } return &Client{ spec: spec, - csv: client, filetype: client, }, nil @@ -61,7 +56,6 @@ func NewClient(spec *FileSpec) (*Client, error) { } return &Client{ spec: spec, - json: client, filetype: client, }, nil @@ -70,11 +64,7 @@ func NewClient(spec *FileSpec) (*Client, error) { if err != nil { return &Client{}, err } - return &Client{ - spec: spec, - parquet: client, - filetype: client, - }, nil + return &Client{spec: spec, filetype: client}, nil default: panic("unknown format " + spec.Format) diff --git a/csv/read.go b/csv/read.go index 70c4468e..505f049d 100644 --- a/csv/read.go +++ b/csv/read.go @@ -2,16 +2,16 @@ package csv import ( "fmt" - "io" "github.com/apache/arrow/go/v14/arrow" "github.com/apache/arrow/go/v14/arrow/array" "github.com/apache/arrow/go/v14/arrow/csv" "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/cloudquery/filetypes/v4/types" "github.com/cloudquery/plugin-sdk/v4/schema" ) -func (cl *Client) Read(r io.Reader, table *schema.Table, res chan<- arrow.Record) error { +func (cl *Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { arrowSchema := table.ToArrowSchema() newSchema := convertSchema(arrowSchema) reader := csv.NewReader(r, newSchema, diff --git a/json/read.go b/json/read.go index 1e6e97d4..12bc9236 100644 --- a/json/read.go +++ b/json/read.go @@ -2,17 +2,17 @@ package json import ( "bufio" - "io" "github.com/apache/arrow/go/v14/arrow" "github.com/apache/arrow/go/v14/arrow/array" "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/cloudquery/filetypes/v4/types" "github.com/cloudquery/plugin-sdk/v4/schema" ) const maxJSONSize = 1024 * 1024 * 20 -func (*Client) Read(r io.Reader, table *schema.Table, res chan<- arrow.Record) error { +func (*Client) Read(r types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) rb := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) diff --git a/parquet/read.go b/parquet/read.go index e649933d..4d1f403d 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -8,13 +8,13 @@ import ( "github.com/apache/arrow/go/v14/arrow" "github.com/apache/arrow/go/v14/arrow/array" "github.com/apache/arrow/go/v14/arrow/memory" - "github.com/apache/arrow/go/v14/parquet" "github.com/apache/arrow/go/v14/parquet/file" "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/cloudquery/filetypes/v4/types" "github.com/cloudquery/plugin-sdk/v4/schema" ) -func (*Client) Read(f parquet.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { +func (*Client) Read(f types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { ctx := context.Background() rdr, err := file.NewParquetReader(f) if err != nil { diff --git a/read.go b/read.go index 3fc74fe4..47982e0d 100644 --- a/read.go +++ b/read.go @@ -6,16 +6,11 @@ import ( "io" "github.com/apache/arrow/go/v14/arrow" + "github.com/cloudquery/filetypes/v4/types" "github.com/cloudquery/plugin-sdk/v4/schema" ) -type ReaderAtSeeker interface { - io.Reader - io.ReaderAt - io.Seeker -} - -func (cl *Client) Read(f ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { +func (cl *Client) Read(f types.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { if cl.spec.Compression == CompressionTypeGZip { rr, err := gzip.NewReader(f) if err != nil { @@ -29,21 +24,5 @@ func (cl *Client) Read(f ReaderAtSeeker, table *schema.Table, res chan<- arrow.R f = bytes.NewReader(b) } - switch cl.spec.Format { - case FormatTypeCSV: - if err := cl.csv.Read(f, table, res); err != nil { - return err - } - case FormatTypeJSON: - if err := cl.json.Read(f, table, res); err != nil { - return err - } - case FormatTypeParquet: - if err := cl.parquet.Read(f, table, res); err != nil { - return err - } - default: - panic("unknown format " + cl.spec.Format) - } - return nil + return cl.filetype.Read(f, table, res) } diff --git a/spec.go b/spec.go index ca43ef3a..19c28171 100644 --- a/spec.go +++ b/spec.go @@ -36,7 +36,6 @@ type FileSpec struct { } func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { - } func (s *FileSpec) SetDefaults() { diff --git a/types/types.go b/types/types.go index acb9d501..430ca514 100644 --- a/types/types.go +++ b/types/types.go @@ -7,8 +7,15 @@ import ( "github.com/cloudquery/plugin-sdk/v4/schema" ) +type ReaderAtSeeker interface { + io.Reader + io.ReaderAt + io.Seeker +} + type FileType interface { WriteHeader(io.Writer, *schema.Table) (Handle, error) + Read(r ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error } type Handle interface { From 501994b6775cf8b54266e0a2a4e0bff325917237 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 16:06:38 +0300 Subject: [PATCH 03/12] optimize more --- client.go | 29 ++++++++--------------------- stream.go | 4 ++-- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/client.go b/client.go index c366a48f..703482df 100644 --- a/client.go +++ b/client.go @@ -31,6 +31,7 @@ func NewClient(spec *FileSpec) (*Client, error) { return &Client{}, err } + var client types.FileType switch spec.Format { case FormatTypeCSV: opts := []csvFile.Options{ @@ -40,33 +41,19 @@ func NewClient(spec *FileSpec) (*Client, error) { opts = append(opts, csvFile.WithHeader()) } - client, err := csvFile.NewClient(opts...) - if err != nil { - return &Client{}, err - } - return &Client{ - spec: spec, - filetype: client, - }, nil + client, err = csvFile.NewClient(opts...) case FormatTypeJSON: - client, err := jsonFile.NewClient() - if err != nil { - return &Client{}, err - } - return &Client{ - spec: spec, - filetype: client, - }, nil + client, err = jsonFile.NewClient() case FormatTypeParquet: - client, err := parquet.NewClient(parquet.WithSpec(*spec.parquetSpec)) - if err != nil { - return &Client{}, err - } - return &Client{spec: spec, filetype: client}, nil + client, err = parquet.NewClient(parquet.WithSpec(*spec.parquetSpec)) default: panic("unknown format " + spec.Format) } + if err != nil { + return nil, err + } + return &Client{spec: spec, filetype: client}, nil } diff --git a/stream.go b/stream.go index 6a2c0eaf..479df14a 100644 --- a/stream.go +++ b/stream.go @@ -27,7 +27,7 @@ func (w *writeCloser) Close() error { } // StartStream starts a streaming upload using the provided uploadFunc. -func (c *Client) StartStream(table *schema.Table, uploadFunc func(io.Reader) error) (*Stream, error) { +func (cl *Client) StartStream(table *schema.Table, uploadFunc func(io.Reader) error) (*Stream, error) { pr, pw := io.Pipe() doneCh := make(chan error) @@ -39,7 +39,7 @@ func (c *Client) StartStream(table *schema.Table, uploadFunc func(io.Reader) err }() wc := &writeCloser{PipeWriter: pw} - h, err := c.WriteHeader(wc, table) + h, err := cl.WriteHeader(wc, table) if err != nil { _ = pw.CloseWithError(err) <-doneCh From 0978748e10aab1ede701a1f97ee38e5b8faa61bd Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 17:04:36 +0300 Subject: [PATCH 04/12] gen format_spec --- csv/spec.go | 27 +++++++++++++++++++- json/spec.go | 10 ++++++++ parquet/spec.go | 10 ++++++++ schema.json | 65 ++++++++++++++++++++++++++++++++++++++++++++++--- spec.go | 21 +++++++++++++--- 5 files changed, 124 insertions(+), 9 deletions(-) diff --git a/csv/spec.go b/csv/spec.go index 755ef241..bc85585d 100644 --- a/csv/spec.go +++ b/csv/spec.go @@ -1,12 +1,37 @@ package csv -import "fmt" +import ( + "fmt" + + "github.com/invopop/jsonschema" +) type Spec struct { SkipHeader bool `json:"skip_header,omitempty"` Delimiter string `json:"delimiter,omitempty"` } +func (Spec) JSONSchema() *jsonschema.Schema { + properties := jsonschema.NewProperties() + properties.Set("skip_header", &jsonschema.Schema{ + Type: "boolean", + Description: "Specifies if the first line of a file should be the header.", + Default: false, + }) + properties.Set("delimiter", &jsonschema.Schema{ + Type: "string", + Description: "Character that will be used as want to use as the delimiter.", + Pattern: `^.$`, // a single character + Default: ",", + }) + return &jsonschema.Schema{ + Description: "CloudQuery CSV file output spec", + Properties: properties, + Type: "object", + AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false + } +} + func (s *Spec) SetDefaults() { if s.Delimiter == "" { s.Delimiter = "," diff --git a/json/spec.go b/json/spec.go index 377a8bb0..fb79f2ec 100644 --- a/json/spec.go +++ b/json/spec.go @@ -1,7 +1,17 @@ package json +import "github.com/invopop/jsonschema" + type Spec struct{} +func (Spec) JSONSchema() *jsonschema.Schema { + return &jsonschema.Schema{ + Description: "CloudQuery JSON file output spec", + Type: "object", + AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false + } +} + func (*Spec) SetDefaults() {} func (*Spec) Validate() error { diff --git a/parquet/spec.go b/parquet/spec.go index 0e296807..a3f71a77 100644 --- a/parquet/spec.go +++ b/parquet/spec.go @@ -1,7 +1,17 @@ package parquet +import "github.com/invopop/jsonschema" + type Spec struct{} +func (Spec) JSONSchema() *jsonschema.Schema { + return &jsonschema.Schema{ + Description: "CloudQuery Parquet file output spec", + Type: "object", + AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false + } +} + func (*Spec) SetDefaults() { } diff --git a/schema.json b/schema.json index 6d4b06d9..269b394c 100644 --- a/schema.json +++ b/schema.json @@ -4,24 +4,81 @@ "$ref": "#/$defs/FileSpec", "$defs": { "FileSpec": { + "$defs": { + "CSV": { + "properties": { + "skip_header": { + "type": "boolean", + "description": "Specifies if the first line of a file should be the header.", + "default": false + }, + "delimiter": { + "type": "string", + "pattern": "^.$", + "description": "Character that will be used as want to use as the delimiter.", + "default": "," + } + }, + "additionalProperties": false, + "type": "object", + "description": "CloudQuery CSV file output spec" + }, + "JSON": { + "additionalProperties": false, + "type": "object", + "description": "CloudQuery JSON file output spec" + }, + "Parquet": { + "additionalProperties": false, + "type": "object", + "description": "CloudQuery Parquet file output spec" + } + }, "properties": { "format": { - "type": "string" + "type": "string", + "enum": [ + "csv", + "json", + "parquet" + ], + "description": "Output format." }, "format_spec": { "oneOf": [ - true, + { + "oneOf": [ + { + "$ref": "CSV" + }, + { + "$ref": "JSON" + }, + { + "$ref": "Parquet" + } + ], + "description": "Format spec." + }, { "type": "null" } ] }, "compression": { - "type": "string" + "type": "string", + "enum": [ + "", + "gzip" + ], + "description": "Compression type.\nEmpty or missing stands for no compression." } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "format" + ] } } } diff --git a/spec.go b/spec.go index 19c28171..63183aa5 100644 --- a/spec.go +++ b/spec.go @@ -19,6 +19,7 @@ const ( FormatTypeParquet = "parquet" ) +// Compression type. type CompressionType string const ( @@ -27,15 +28,28 @@ const ( ) type FileSpec struct { - Format FormatType `json:"format,omitempty"` - FormatSpec any `json:"format_spec,omitempty"` - Compression CompressionType `json:"compression,omitempty"` + // Output format. + Format FormatType `json:"format,omitempty" jsonschema:"required,enum=csv,enum=json,enum=parquet"` + + // Format spec. + FormatSpec any `json:"format_spec,omitempty" jsonschema:"oneof_ref=CSV;JSON;Parquet"` + + // Compression type. + // Empty or missing stands for no compression. + Compression CompressionType `json:"compression,omitempty" jsonschema:"enum=,enum=gzip"` + csvSpec *csv.Spec jsonSpec *jsonFile.Spec parquetSpec *parquet.Spec } func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { + if sc.Definitions == nil { + sc.Definitions = make(jsonschema.Definitions) + } + sc.Definitions["CSV"] = csv.Spec{}.JSONSchema() + sc.Definitions["JSON"] = jsonFile.Spec{}.JSONSchema() + sc.Definitions["Parquet"] = parquet.Spec{}.JSONSchema() } func (s *FileSpec) SetDefaults() { @@ -78,7 +92,6 @@ func (s *FileSpec) UnmarshalSpec() error { return err } dec := json.NewDecoder(bytes.NewReader(b)) - dec.UseNumber() dec.DisallowUnknownFields() switch s.Format { From 58e337ef4aa376ffbe64811d9dd9616b21033557 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 17:12:59 +0300 Subject: [PATCH 05/12] tests for type specs --- csv/spec_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++++ json/spec_test.go | 25 ++++++++++++++++ parquet/spec_test.go | 25 ++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 csv/spec_test.go create mode 100644 json/spec_test.go create mode 100644 parquet/spec_test.go diff --git a/csv/spec_test.go b/csv/spec_test.go new file mode 100644 index 00000000..a274d56a --- /dev/null +++ b/csv/spec_test.go @@ -0,0 +1,71 @@ +package csv + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/stretchr/testify/require" +) + +func TestSpec_JSONSchema(t *testing.T) { + schema, err := jsonschema.Generate(Spec{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "extra keyword", + Err: true, + Spec: `{"extra":true}`, + }, + { + Name: "skip_header:true", + Spec: `{"skip_header":true}`, + }, + { + Name: "skip_header:false", + Spec: `{"skip_header":false}`, + }, + { + Name: "null skip_header", + Err: true, + Spec: `{"skip_header":null}`, + }, + { + Name: "bad skip_header", + Err: true, + Spec: `{"skip_header":123}`, + }, + { + Name: "empty delimiter", + Err: true, + Spec: `{"delimiter":""}`, + }, + { + Name: "null delimiter", + Err: true, + Spec: `{"delimiter":null}`, + }, + { + Name: "bad delimiter", + Err: true, + Spec: `{"delimiter":123}`, + }, + { + Name: "delimiter:\",,\"", + Err: true, + Spec: `{"delimiter":",,"}`, + }, + { + Name: "tab delimiter", + Spec: `{"delimiter":"\t"}`, + }, + { + Name: "space delimiter", + Spec: `{"delimiter":" "}`, + }, + }) +} diff --git a/json/spec_test.go b/json/spec_test.go new file mode 100644 index 00000000..7dbe6dc6 --- /dev/null +++ b/json/spec_test.go @@ -0,0 +1,25 @@ +package json + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/stretchr/testify/require" +) + +func TestSpec_JSONSchema(t *testing.T) { + schema, err := jsonschema.Generate(Spec{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "extra keyword", + Err: true, + Spec: `{"extra":true}`, + }, + }) +} diff --git a/parquet/spec_test.go b/parquet/spec_test.go new file mode 100644 index 00000000..a1436978 --- /dev/null +++ b/parquet/spec_test.go @@ -0,0 +1,25 @@ +package parquet + +import ( + "testing" + + "github.com/cloudquery/codegen/jsonschema" + "github.com/stretchr/testify/require" +) + +func TestSpec_JSONSchema(t *testing.T) { + schema, err := jsonschema.Generate(Spec{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Spec: `{}`, + }, + { + Name: "extra keyword", + Err: true, + Spec: `{"extra":true}`, + }, + }) +} From fbeba11c9a3fe31553538f1aa7d09498db56b5f4 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 17:27:02 +0300 Subject: [PATCH 06/12] conditional types --- schema.json | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ spec.go | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/schema.json b/schema.json index 269b394c..01ffdb45 100644 --- a/schema.json +++ b/schema.json @@ -34,6 +34,80 @@ "description": "CloudQuery Parquet file output spec" } }, + "allOf": [ + { + "if": { + "properties": { + "format": { + "type": "string", + "const": "csv" + } + } + }, + "then": { + "properties": { + "format_spec": { + "oneOf": [ + { + "$ref": "#/$defs/CSV" + }, + { + "type": "null" + } + ] + } + } + } + }, + { + "if": { + "properties": { + "format": { + "type": "string", + "const": "json" + } + } + }, + "then": { + "properties": { + "format_spec": { + "oneOf": [ + { + "$ref": "#/$defs/JSON" + }, + { + "type": "null" + } + ] + } + } + } + }, + { + "if": { + "properties": { + "format": { + "type": "string", + "const": "parquet" + } + } + }, + "then": { + "properties": { + "format_spec": { + "oneOf": [ + { + "$ref": "#/$defs/Parquet" + }, + { + "type": "null" + } + ] + } + } + } + } + ], "properties": { "format": { "type": "string", diff --git a/spec.go b/spec.go index 63183aa5..409ec511 100644 --- a/spec.go +++ b/spec.go @@ -9,6 +9,7 @@ import ( jsonFile "github.com/cloudquery/filetypes/v4/json" "github.com/cloudquery/filetypes/v4/parquet" "github.com/invopop/jsonschema" + orderedmap "github.com/wk8/go-ordered-map/v2" ) type FormatType string @@ -50,6 +51,82 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { sc.Definitions["CSV"] = csv.Spec{}.JSONSchema() sc.Definitions["JSON"] = jsonFile.Spec{}.JSONSchema() sc.Definitions["Parquet"] = parquet.Spec{}.JSONSchema() + + // now we need to enforce format -> specific type + sc.AllOf = []*jsonschema.Schema{ + // CSV + { + If: &jsonschema.Schema{ + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeCSV}) + return properties + }(), + }, + Then: &jsonschema.Schema{ + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Ref: jsonschema.ID("").Def("CSV").String(), + }, + {Type: "null"}, + }, + }) + return properties + }(), + }, + }, + // JSON + { + If: &jsonschema.Schema{ + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeJSON}) + return properties + }(), + }, + Then: &jsonschema.Schema{ + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Ref: jsonschema.ID("").Def("JSON").String(), + }, + {Type: "null"}, + }, + }) + return properties + }(), + }, + }, + // Parquet + { + If: &jsonschema.Schema{ + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeParquet}) + return properties + }(), + }, + Then: &jsonschema.Schema{ + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Ref: jsonschema.ID("").Def("Parquet").String(), + }, + {Type: "null"}, + }, + }) + return properties + }(), + }, + }, + } } func (s *FileSpec) SetDefaults() { From ab08470ef1d51a14deea72e1ab6b97175098447a Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 20 Oct 2023 18:49:13 +0300 Subject: [PATCH 07/12] use oneOf --- schema.json | 104 ++++++++++++++++++++++------------------------------ spec.go | 98 +++++++++++++++++++------------------------------ 2 files changed, 80 insertions(+), 122 deletions(-) diff --git a/schema.json b/schema.json index 01ffdb45..235f8a3d 100644 --- a/schema.json +++ b/schema.json @@ -34,76 +34,58 @@ "description": "CloudQuery Parquet file output spec" } }, - "allOf": [ + "oneOf": [ { - "if": { - "properties": { - "format": { - "type": "string", - "const": "csv" - } - } - }, - "then": { - "properties": { - "format_spec": { - "oneOf": [ - { - "$ref": "#/$defs/CSV" - }, - { - "type": "null" - } - ] - } + "properties": { + "format": { + "type": "string", + "const": "csv" + }, + "format_spec": { + "oneOf": [ + { + "$ref": "#/$defs/CSV" + }, + { + "type": "null" + } + ] } } }, { - "if": { - "properties": { - "format": { - "type": "string", - "const": "json" - } - } - }, - "then": { - "properties": { - "format_spec": { - "oneOf": [ - { - "$ref": "#/$defs/JSON" - }, - { - "type": "null" - } - ] - } + "properties": { + "format": { + "type": "string", + "const": "json" + }, + "format_spec": { + "oneOf": [ + { + "$ref": "#/$defs/JSON" + }, + { + "type": "null" + } + ] } } }, { - "if": { - "properties": { - "format": { - "type": "string", - "const": "parquet" - } - } - }, - "then": { - "properties": { - "format_spec": { - "oneOf": [ - { - "$ref": "#/$defs/Parquet" - }, - { - "type": "null" - } - ] - } + "properties": { + "format": { + "type": "string", + "const": "parquet" + }, + "format_spec": { + "oneOf": [ + { + "$ref": "#/$defs/Parquet" + }, + { + "type": "null" + } + ] } } } diff --git a/spec.go b/spec.go index 409ec511..074281f8 100644 --- a/spec.go +++ b/spec.go @@ -53,78 +53,54 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { sc.Definitions["Parquet"] = parquet.Spec{}.JSONSchema() // now we need to enforce format -> specific type - sc.AllOf = []*jsonschema.Schema{ + sc.OneOf = []*jsonschema.Schema{ // CSV { - If: &jsonschema.Schema{ - Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { - properties := jsonschema.NewProperties() - properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeCSV}) - return properties - }(), - }, - Then: &jsonschema.Schema{ - Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { - properties := jsonschema.NewProperties() - properties.Set("format_spec", &jsonschema.Schema{ - OneOf: []*jsonschema.Schema{ - { - Ref: jsonschema.ID("").Def("CSV").String(), - }, - {Type: "null"}, + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeCSV}) + properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Ref: jsonschema.ID("").Def("CSV").String(), }, - }) - return properties - }(), - }, + {Type: "null"}, + }, + }) + return properties + }(), }, // JSON { - If: &jsonschema.Schema{ - Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { - properties := jsonschema.NewProperties() - properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeJSON}) - return properties - }(), - }, - Then: &jsonschema.Schema{ - Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { - properties := jsonschema.NewProperties() - properties.Set("format_spec", &jsonschema.Schema{ - OneOf: []*jsonschema.Schema{ - { - Ref: jsonschema.ID("").Def("JSON").String(), - }, - {Type: "null"}, + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeJSON}) + properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Ref: jsonschema.ID("").Def("JSON").String(), }, - }) - return properties - }(), - }, + {Type: "null"}, + }, + }) + return properties + }(), }, // Parquet { - If: &jsonschema.Schema{ - Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { - properties := jsonschema.NewProperties() - properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeParquet}) - return properties - }(), - }, - Then: &jsonschema.Schema{ - Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { - properties := jsonschema.NewProperties() - properties.Set("format_spec", &jsonschema.Schema{ - OneOf: []*jsonschema.Schema{ - { - Ref: jsonschema.ID("").Def("Parquet").String(), - }, - {Type: "null"}, + Properties: func() *orderedmap.OrderedMap[string, *jsonschema.Schema] { + properties := jsonschema.NewProperties() + properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeParquet}) + properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Ref: jsonschema.ID("").Def("Parquet").String(), }, - }) - return properties - }(), - }, + {Type: "null"}, + }, + }) + return properties + }(), }, } } From 244feac338679a868a830f91fd835c2f4b9bcef5 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Sat, 21 Oct 2023 09:10:24 +0300 Subject: [PATCH 08/12] try anchors --- csv/spec.go | 1 + json/spec.go | 1 + parquet/spec.go | 1 + schema.json | 9 ++++++--- spec.go | 12 +++--------- spec_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 59 insertions(+), 12 deletions(-) diff --git a/csv/spec.go b/csv/spec.go index bc85585d..e8267ee2 100644 --- a/csv/spec.go +++ b/csv/spec.go @@ -25,6 +25,7 @@ func (Spec) JSONSchema() *jsonschema.Schema { Default: ",", }) return &jsonschema.Schema{ + Anchor: "csv-spec", Description: "CloudQuery CSV file output spec", Properties: properties, Type: "object", diff --git a/json/spec.go b/json/spec.go index fb79f2ec..bd593e7b 100644 --- a/json/spec.go +++ b/json/spec.go @@ -6,6 +6,7 @@ type Spec struct{} func (Spec) JSONSchema() *jsonschema.Schema { return &jsonschema.Schema{ + Anchor: "json-spec", Description: "CloudQuery JSON file output spec", Type: "object", AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false diff --git a/parquet/spec.go b/parquet/spec.go index a3f71a77..fc44a86b 100644 --- a/parquet/spec.go +++ b/parquet/spec.go @@ -6,6 +6,7 @@ type Spec struct{} func (Spec) JSONSchema() *jsonschema.Schema { return &jsonschema.Schema{ + Anchor: "parquet-spec", Description: "CloudQuery Parquet file output spec", Type: "object", AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false diff --git a/schema.json b/schema.json index 235f8a3d..4c4d9640 100644 --- a/schema.json +++ b/schema.json @@ -6,6 +6,7 @@ "FileSpec": { "$defs": { "CSV": { + "$anchor": "csv-spec", "properties": { "skip_header": { "type": "boolean", @@ -24,11 +25,13 @@ "description": "CloudQuery CSV file output spec" }, "JSON": { + "$anchor": "json-spec", "additionalProperties": false, "type": "object", "description": "CloudQuery JSON file output spec" }, "Parquet": { + "$anchor": "parquet-spec", "additionalProperties": false, "type": "object", "description": "CloudQuery Parquet file output spec" @@ -44,7 +47,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#/$defs/CSV" + "$ref": "#csv-spec" }, { "type": "null" @@ -62,7 +65,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#/$defs/JSON" + "$ref": "#json-spec" }, { "type": "null" @@ -80,7 +83,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#/$defs/Parquet" + "$ref": "#parquet-spec" }, { "type": "null" diff --git a/spec.go b/spec.go index 074281f8..9e7229d5 100644 --- a/spec.go +++ b/spec.go @@ -61,9 +61,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeCSV}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - { - Ref: jsonschema.ID("").Def("CSV").String(), - }, + {Ref: "#csv-spec"}, {Type: "null"}, }, }) @@ -77,9 +75,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeJSON}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - { - Ref: jsonschema.ID("").Def("JSON").String(), - }, + {Ref: "#json-spec"}, {Type: "null"}, }, }) @@ -93,9 +89,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeParquet}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - { - Ref: jsonschema.ID("").Def("Parquet").String(), - }, + {Ref: "#parquet-spec"}, {Type: "null"}, }, }) diff --git a/spec_test.go b/spec_test.go index 54e593ca..b2a93a44 100644 --- a/spec_test.go +++ b/spec_test.go @@ -3,9 +3,11 @@ package filetypes import ( "testing" + "github.com/cloudquery/codegen/jsonschema" "github.com/cloudquery/filetypes/v4/csv" "github.com/cloudquery/filetypes/v4/json" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSpecMethods(t *testing.T) { @@ -102,3 +104,48 @@ func TestSpecMethods(t *testing.T) { assert.Equal(t, tc.postDefaultsJSON, tc.FileSpec.jsonSpec) } } + +func TestFileSpec_JSONSchemaExtend(t *testing.T) { + schema, err := jsonschema.Generate(FileSpec{}) + require.NoError(t, err) + + jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{ + { + Name: "empty", + Err: true, // missing format + Spec: `{}`, + }, + { + Name: "empty format", + Err: true, + Spec: `{"format":""}`, + }, + { + Name: "null format", + Err: true, + Spec: `{"format":null}`, + }, + { + Name: "bad format", + Err: true, + Spec: `{"format":123}`, + }, + { + Name: "bad format value", + Err: true, + Spec: `{"format":"abc"}`, + }, + { + Name: "csv format", + Spec: `{"format":"csv"}`, + }, + { + Name: "json format", + Spec: `{"format":"json"}`, + }, + { + Name: "parquet format", + Spec: `{"format":"parquet"}`, + }, + }) +} From 0cea90377a3545d24097df1bc8111858b4dc414c Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Sat, 21 Oct 2023 19:50:06 +0300 Subject: [PATCH 09/12] fix schema --- csv/spec.go | 5 ++--- json/spec.go | 3 +-- parquet/spec.go | 3 +-- schema.json | 35 ++++++++++++++++------------------- spec.go | 31 ++++++++++++++++++++++--------- spec_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 80 insertions(+), 35 deletions(-) diff --git a/csv/spec.go b/csv/spec.go index e8267ee2..e1f0b7dc 100644 --- a/csv/spec.go +++ b/csv/spec.go @@ -20,13 +20,12 @@ func (Spec) JSONSchema() *jsonschema.Schema { }) properties.Set("delimiter", &jsonschema.Schema{ Type: "string", - Description: "Character that will be used as want to use as the delimiter.", + Description: "Character that will be used as the delimiter.", Pattern: `^.$`, // a single character Default: ",", }) return &jsonschema.Schema{ - Anchor: "csv-spec", - Description: "CloudQuery CSV file output spec", + Description: "CloudQuery CSV file output spec.", Properties: properties, Type: "object", AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false diff --git a/json/spec.go b/json/spec.go index bd593e7b..6024245c 100644 --- a/json/spec.go +++ b/json/spec.go @@ -6,8 +6,7 @@ type Spec struct{} func (Spec) JSONSchema() *jsonschema.Schema { return &jsonschema.Schema{ - Anchor: "json-spec", - Description: "CloudQuery JSON file output spec", + Description: "CloudQuery JSON file output spec.", Type: "object", AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false } diff --git a/parquet/spec.go b/parquet/spec.go index fc44a86b..272b014f 100644 --- a/parquet/spec.go +++ b/parquet/spec.go @@ -6,8 +6,7 @@ type Spec struct{} func (Spec) JSONSchema() *jsonschema.Schema { return &jsonschema.Schema{ - Anchor: "parquet-spec", - Description: "CloudQuery Parquet file output spec", + Description: "CloudQuery Parquet file output spec.", Type: "object", AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false } diff --git a/schema.json b/schema.json index 4c4d9640..50aaaf96 100644 --- a/schema.json +++ b/schema.json @@ -4,9 +4,9 @@ "$ref": "#/$defs/FileSpec", "$defs": { "FileSpec": { + "$id": "/schemas/FileSpec", "$defs": { - "CSV": { - "$anchor": "csv-spec", + "csv": { "properties": { "skip_header": { "type": "boolean", @@ -16,25 +16,23 @@ "delimiter": { "type": "string", "pattern": "^.$", - "description": "Character that will be used as want to use as the delimiter.", + "description": "Character that will be used as the delimiter.", "default": "," } }, "additionalProperties": false, "type": "object", - "description": "CloudQuery CSV file output spec" + "description": "CloudQuery CSV file output spec." }, - "JSON": { - "$anchor": "json-spec", + "json": { "additionalProperties": false, "type": "object", - "description": "CloudQuery JSON file output spec" + "description": "CloudQuery JSON file output spec." }, - "Parquet": { - "$anchor": "parquet-spec", + "parquet": { "additionalProperties": false, "type": "object", - "description": "CloudQuery Parquet file output spec" + "description": "CloudQuery Parquet file output spec." } }, "oneOf": [ @@ -47,7 +45,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#csv-spec" + "$ref": "#/$defs/csv" }, { "type": "null" @@ -65,7 +63,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#json-spec" + "$ref": "#/$defs/json" }, { "type": "null" @@ -83,7 +81,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#parquet-spec" + "$ref": "#/$defs/parquet" }, { "type": "null" @@ -106,18 +104,17 @@ "format_spec": { "oneOf": [ { - "oneOf": [ + "anyOf": [ { - "$ref": "CSV" + "$ref": "#/$defs/csv" }, { - "$ref": "JSON" + "$ref": "#/$defs/json" }, { - "$ref": "Parquet" + "$ref": "#/$defs/parquet" } - ], - "description": "Format spec." + ] }, { "type": "null" diff --git a/spec.go b/spec.go index 9e7229d5..eddb0b22 100644 --- a/spec.go +++ b/spec.go @@ -33,7 +33,7 @@ type FileSpec struct { Format FormatType `json:"format,omitempty" jsonschema:"required,enum=csv,enum=json,enum=parquet"` // Format spec. - FormatSpec any `json:"format_spec,omitempty" jsonschema:"oneof_ref=CSV;JSON;Parquet"` + FormatSpec any `json:"format_spec,omitempty"` // Compression type. // Empty or missing stands for no compression. @@ -45,12 +45,25 @@ type FileSpec struct { } func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { - if sc.Definitions == nil { - sc.Definitions = make(jsonschema.Definitions) + sc.ID = "/schemas/FileSpec" + sc.Definitions = jsonschema.Definitions{ + "csv": csv.Spec{}.JSONSchema(), + "json": jsonFile.Spec{}.JSONSchema(), + "parquet": parquet.Spec{}.JSONSchema(), } - sc.Definitions["CSV"] = csv.Spec{}.JSONSchema() - sc.Definitions["JSON"] = jsonFile.Spec{}.JSONSchema() - sc.Definitions["Parquet"] = parquet.Spec{}.JSONSchema() + + sc.Properties.Set("format_spec", &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + AnyOf: []*jsonschema.Schema{ + {Ref: jsonschema.EmptyID.Def("csv").String()}, + {Ref: jsonschema.EmptyID.Def("json").String()}, + {Ref: jsonschema.EmptyID.Def("parquet").String()}, + }, + }, + {Type: "null"}, + }, + }) // now we need to enforce format -> specific type sc.OneOf = []*jsonschema.Schema{ @@ -61,7 +74,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeCSV}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - {Ref: "#csv-spec"}, + {Ref: jsonschema.EmptyID.Def("csv").String()}, {Type: "null"}, }, }) @@ -75,7 +88,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeJSON}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - {Ref: "#json-spec"}, + {Ref: jsonschema.EmptyID.Def("json").String()}, {Type: "null"}, }, }) @@ -89,7 +102,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeParquet}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - {Ref: "#parquet-spec"}, + {Ref: jsonschema.EmptyID.Def("parquet").String()}, {Type: "null"}, }, }) diff --git a/spec_test.go b/spec_test.go index b2a93a44..c0dc8ac0 100644 --- a/spec_test.go +++ b/spec_test.go @@ -139,13 +139,51 @@ func TestFileSpec_JSONSchemaExtend(t *testing.T) { Name: "csv format", Spec: `{"format":"csv"}`, }, + { + Name: "csv format + empty format_spec", + Spec: `{"format":"csv","format_spec":{}}`, + }, + { + Name: "csv format + null format_spec", + Spec: `{"format":"csv","format_spec":null}`, + }, + { + Name: "csv format + csv format_spec", + Spec: `{"format":"csv","format_spec":{"skip_header": true, "delimiter":","}}`, + }, { Name: "json format", Spec: `{"format":"json"}`, }, + { + Name: "json format + empty format_spec", + Spec: `{"format":"json","format_spec":{}}`, + }, + { + Name: "json format + null format_spec", + Spec: `{"format":"json","format_spec":null}`, + }, + { + Name: "json format + csv format_spec", + Err: true, + Spec: `{"format":"json","format_spec":{"skip_header": true, "delimiter":","}}`, + }, { Name: "parquet format", Spec: `{"format":"parquet"}`, }, + { + Name: "parquet format + empty format_spec", + Spec: `{"format":"parquet","format_spec":{}}`, + }, + { + Name: "parquet format + null format_spec", + Spec: `{"format":"parquet","format_spec":null}`, + }, + { + Name: "parquet format + csv format_spec", + Err: true, + Spec: `{"format":"parquet","format_spec":{"skip_header": true, "delimiter":","}}`, + }, }) } From 96e96725d43606645c87dbebd5c4f5b1a3a3f479 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Mon, 23 Oct 2023 12:04:13 +0300 Subject: [PATCH 10/12] use titles for defs --- go.mod | 4 ++-- schema.json | 18 +++++++++--------- spec.go | 18 +++++++++--------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 81ca72eb..30e4837b 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/google/uuid v1.3.1 github.com/invopop/jsonschema v0.11.0 github.com/stretchr/testify v1.8.4 + github.com/wk8/go-ordered-map/v2 v2.1.8 ) require ( @@ -36,13 +37,12 @@ require ( github.com/rs/zerolog v1.31.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/thoas/go-funk v0.9.3 // indirect - github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.13.0 // indirect; indirect // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/schema.json b/schema.json index 50aaaf96..e0a6bca6 100644 --- a/schema.json +++ b/schema.json @@ -6,7 +6,7 @@ "FileSpec": { "$id": "/schemas/FileSpec", "$defs": { - "csv": { + "CSVSpec": { "properties": { "skip_header": { "type": "boolean", @@ -24,12 +24,12 @@ "type": "object", "description": "CloudQuery CSV file output spec." }, - "json": { + "JSONSpec": { "additionalProperties": false, "type": "object", "description": "CloudQuery JSON file output spec." }, - "parquet": { + "ParquetSpec": { "additionalProperties": false, "type": "object", "description": "CloudQuery Parquet file output spec." @@ -45,7 +45,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#/$defs/csv" + "$ref": "#/$defs/CSVSpec" }, { "type": "null" @@ -63,7 +63,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#/$defs/json" + "$ref": "#/$defs/JSONSpec" }, { "type": "null" @@ -81,7 +81,7 @@ "format_spec": { "oneOf": [ { - "$ref": "#/$defs/parquet" + "$ref": "#/$defs/ParquetSpec" }, { "type": "null" @@ -106,13 +106,13 @@ { "anyOf": [ { - "$ref": "#/$defs/csv" + "$ref": "#/$defs/CSVSpec" }, { - "$ref": "#/$defs/json" + "$ref": "#/$defs/JSONSpec" }, { - "$ref": "#/$defs/parquet" + "$ref": "#/$defs/ParquetSpec" } ] }, diff --git a/spec.go b/spec.go index eddb0b22..11e55af2 100644 --- a/spec.go +++ b/spec.go @@ -47,18 +47,18 @@ type FileSpec struct { func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { sc.ID = "/schemas/FileSpec" sc.Definitions = jsonschema.Definitions{ - "csv": csv.Spec{}.JSONSchema(), - "json": jsonFile.Spec{}.JSONSchema(), - "parquet": parquet.Spec{}.JSONSchema(), + "CSVSpec": csv.Spec{}.JSONSchema(), + "JSONSpec": jsonFile.Spec{}.JSONSchema(), + "ParquetSpec": parquet.Spec{}.JSONSchema(), } sc.Properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ { AnyOf: []*jsonschema.Schema{ - {Ref: jsonschema.EmptyID.Def("csv").String()}, - {Ref: jsonschema.EmptyID.Def("json").String()}, - {Ref: jsonschema.EmptyID.Def("parquet").String()}, + {Ref: jsonschema.EmptyID.Def("CSVSpec").String()}, + {Ref: jsonschema.EmptyID.Def("JSONSpec").String()}, + {Ref: jsonschema.EmptyID.Def("ParquetSpec").String()}, }, }, {Type: "null"}, @@ -74,7 +74,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeCSV}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - {Ref: jsonschema.EmptyID.Def("csv").String()}, + {Ref: jsonschema.EmptyID.Def("CSVSpec").String()}, {Type: "null"}, }, }) @@ -88,7 +88,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeJSON}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - {Ref: jsonschema.EmptyID.Def("json").String()}, + {Ref: jsonschema.EmptyID.Def("JSONSpec").String()}, {Type: "null"}, }, }) @@ -102,7 +102,7 @@ func (FileSpec) JSONSchemaExtend(sc *jsonschema.Schema) { properties.Set("format", &jsonschema.Schema{Type: "string", Const: FormatTypeParquet}) properties.Set("format_spec", &jsonschema.Schema{ OneOf: []*jsonschema.Schema{ - {Ref: jsonschema.EmptyID.Def("parquet").String()}, + {Ref: jsonschema.EmptyID.Def("ParquetSpec").String()}, {Type: "null"}, }, }) From 09abdab9f0f51ee6faadf4575b462059778cc34a Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Mon, 23 Oct 2023 12:41:50 +0300 Subject: [PATCH 11/12] tidy --- go.sum | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/go.sum b/go.sum index 2c5e3b66..7dc94f46 100644 --- a/go.sum +++ b/go.sum @@ -10,14 +10,12 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c h1:z0ZtXvDjtohLaNHLGl25gyq7ULS0y64CayNvReJJCTc= -github.com/cloudquery/arrow/go/v14 v14.0.0-20231014001145-dbcb1498009c/go.mod h1:EkHPhLB+98ANnPojOy2sUhM0rzYbPEmrtuA9v8aZp/c= +github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561 h1:sXA8imGI4P8EPdycL1w7mzigIHx2KqyntgAqGuTgTi8= +github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= github.com/cloudquery/codegen v0.3.10 h1:xd6g8UjxnMHzpY/TcM4r1YCcQ75tyc2Ehyu8d+yCpm4= github.com/cloudquery/codegen v0.3.10/go.mod h1:ZqCxX9AGSMUnafE3gKmcT+RJCTgFg7izDxJwFSLGRcg= github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8= github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34= -github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561 h1:sXA8imGI4P8EPdycL1w7mzigIHx2KqyntgAqGuTgTi8= -github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= github.com/cloudquery/plugin-sdk/v4 v4.16.1 h1:Ir2fkqsu5htnnI4wTGTmA6wp1LkixZmmbyxRSIsGoxM= github.com/cloudquery/plugin-sdk/v4 v4.16.1/go.mod h1:ujSFEUAp8BmozOee0ljjsPHQfddXJCUTAzCD6sVKsu8= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= From c75501c5127f8f946b4b9d461327a5a226d72501 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Mon, 23 Oct 2023 13:13:03 +0300 Subject: [PATCH 12/12] less diff --- spec.go | 1 + 1 file changed, 1 insertion(+) diff --git a/spec.go b/spec.go index 11e55af2..98e36311 100644 --- a/spec.go +++ b/spec.go @@ -152,6 +152,7 @@ func (s *FileSpec) UnmarshalSpec() error { return err } dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() dec.DisallowUnknownFields() switch s.Format {