feat: Add JSON schema for specs

.github/workflows/unittest.yml

@@ -19,5 +19,11 @@ jobs:
           go-version-file: go.mod
       - run: go mod download
       - run: go build ./...
+      - name: gen
+        if: github.event_name == 'pull_request'
+        run: make gen
+      - name: Fail if generation updated files
+        if: github.event_name == 'pull_request'
+        run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
       - name: Run tests
         run: go test -race ./...

Makefile

@@ -5,3 +5,11 @@ test:
 .PHONY: lint
 lint:
 	golangci-lint run
+
+.PHONY: gen-spec-schema
+gen-spec-schema:
+	go run schemagen/main.go
+
+# All gen targets
+.PHONY: gen
+gen: gen-spec-schema

client.go

@@ -55,7 +55,7 @@ func NewClient(spec *FileSpec) (*Client, error) {
 	}
 
 	if err != nil {
-		return &Client{}, err
+		return nil, err
 	}
 
 	return &Client{spec: spec, filetype: client}, nil

csv/spec.go

@@ -1,12 +1,37 @@
 package csv
 
-import "fmt"
+import (
+	"fmt"
+
+	"github.com/invopop/jsonschema"
+)
 
 type Spec struct {
 	SkipHeader bool   `json:"skip_header,omitempty"`
 	Delimiter  string `json:"delimiter,omitempty"`
 }
 
+func (Spec) JSONSchema() *jsonschema.Schema {
+	properties := jsonschema.NewProperties()
+	properties.Set("skip_header", &jsonschema.Schema{
+		Type:        "boolean",
+		Description: "Specifies if the first line of a file should be the header.",
+		Default:     false,
+	})
+	properties.Set("delimiter", &jsonschema.Schema{
+		Type:        "string",
+		Description: "Character that will be used as the delimiter.",
+		Pattern:     `^.$`, // a single character
+		Default:     ",",
+	})
+	return &jsonschema.Schema{
+		Description:          "CloudQuery CSV file output spec.",
+		Properties:           properties,
+		Type:                 "object",
+		AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
+	}
+}
+
 func (s *Spec) SetDefaults() {
 	if s.Delimiter == "" {
 		s.Delimiter = ","

csv/spec_test.go

@@ -0,0 +1,71 @@
+package csv
+
+import (
+	"testing"
+
+	"github.com/cloudquery/codegen/jsonschema"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpec_JSONSchema(t *testing.T) {
+	schema, err := jsonschema.Generate(Spec{})
+	require.NoError(t, err)
+
+	jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
+		{
+			Name: "empty",
+			Spec: `{}`,
+		},
+		{
+			Name: "extra keyword",
+			Err:  true,
+			Spec: `{"extra":true}`,
+		},
+		{
+			Name: "skip_header:true",
+			Spec: `{"skip_header":true}`,
+		},
+		{
+			Name: "skip_header:false",
+			Spec: `{"skip_header":false}`,
+		},
+		{
+			Name: "null skip_header",
+			Err:  true,
+			Spec: `{"skip_header":null}`,
+		},
+		{
+			Name: "bad skip_header",
+			Err:  true,
+			Spec: `{"skip_header":123}`,
+		},
+		{
+			Name: "empty delimiter",
+			Err:  true,
+			Spec: `{"delimiter":""}`,
+		},
+		{
+			Name: "null delimiter",
+			Err:  true,
+			Spec: `{"delimiter":null}`,
+		},
+		{
+			Name: "bad delimiter",
+			Err:  true,
+			Spec: `{"delimiter":123}`,
+		},
+		{
+			Name: "delimiter:\",,\"",
+			Err:  true,
+			Spec: `{"delimiter":",,"}`,
+		},
+		{
+			Name: "tab delimiter",
+			Spec: `{"delimiter":"\t"}`,
+		},
+		{
+			Name: "space delimiter",
+			Spec: `{"delimiter":" "}`,
+		},
+	})
+}

go.mod

@@ -5,49 +5,55 @@ go 1.21.1
 require (
 	github.com/apache/arrow/go/v14 v14.0.0-20230929201650-00efb06dc0de
 	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
+	github.com/cloudquery/codegen v0.3.10
 	github.com/cloudquery/plugin-sdk/v4 v4.16.1
 	github.com/goccy/go-json v0.10.2
 	github.com/google/uuid v1.3.1
+	github.com/invopop/jsonschema v0.11.0
 	github.com/stretchr/testify v1.8.4
+	github.com/wk8/go-ordered-map/v2 v2.1.8
 )
 
-require (
-	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
-)
-
-// TODO: remove once all updates are merged
-replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561
-
 require (
 	github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/apache/thrift v0.19.0 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v23.5.26+incompatible // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rs/zerolog v1.31.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	github.com/thoas/go-funk v0.9.3 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect; indirect // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
 	google.golang.org/grpc v1.58.2 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+// TODO: remove once all updates are merged
+replace github.com/apache/arrow/go/v14 => github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561
+
+// github.com/cloudquery/jsonschema @ cqmain
+replace github.com/invopop/jsonschema => github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f

go.sum

@@ -4,10 +4,18 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk=
 github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I=
+github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
+github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561 h1:sXA8imGI4P8EPdycL1w7mzigIHx2KqyntgAqGuTgTi8=
 github.com/cloudquery/arrow/go/v14 v14.0.0-20231023001216-f46436fa3561/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY=
+github.com/cloudquery/codegen v0.3.10 h1:xd6g8UjxnMHzpY/TcM4r1YCcQ75tyc2Ehyu8d+yCpm4=
+github.com/cloudquery/codegen v0.3.10/go.mod h1:ZqCxX9AGSMUnafE3gKmcT+RJCTgFg7izDxJwFSLGRcg=
+github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f h1:vmYGxIGDVpmhk0QVeDwXXbAt+SwQcOn4xH1G25pmKP8=
+github.com/cloudquery/jsonschema v0.0.0-20231018073309-6c617a23d42f/go.mod h1:0SoZ/U7yJlNOR+fWsBSeTvTbGXB6DK01tzJ7m2Xfg34=
 github.com/cloudquery/plugin-sdk/v4 v4.16.1 h1:Ir2fkqsu5htnnI4wTGTmA6wp1LkixZmmbyxRSIsGoxM=
 github.com/cloudquery/plugin-sdk/v4 v4.16.1/go.mod h1:ujSFEUAp8BmozOee0ljjsPHQfddXJCUTAzCD6sVKsu8=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -29,6 +37,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
 github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
 github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
 github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
@@ -39,6 +48,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -70,6 +81,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
 github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
+github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
 github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=

json/spec.go

@@ -1,7 +1,17 @@
 package json
 
+import "github.com/invopop/jsonschema"
+
 type Spec struct{}
 
+func (Spec) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		Description:          "CloudQuery JSON file output spec.",
+		Type:                 "object",
+		AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
+	}
+}
+
 func (*Spec) SetDefaults() {}
 
 func (*Spec) Validate() error {

json/spec_test.go

@@ -0,0 +1,25 @@
+package json
+
+import (
+	"testing"
+
+	"github.com/cloudquery/codegen/jsonschema"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpec_JSONSchema(t *testing.T) {
+	schema, err := jsonschema.Generate(Spec{})
+	require.NoError(t, err)
+
+	jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
+		{
+			Name: "empty",
+			Spec: `{}`,
+		},
+		{
+			Name: "extra keyword",
+			Err:  true,
+			Spec: `{"extra":true}`,
+		},
+	})
+}

parquet/spec.go

@@ -1,7 +1,17 @@
 package parquet
 
+import "github.com/invopop/jsonschema"
+
 type Spec struct{}
 
+func (Spec) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		Description:          "CloudQuery Parquet file output spec.",
+		Type:                 "object",
+		AdditionalProperties: jsonschema.FalseSchema, // "additionalProperties": false
+	}
+}
+
 func (*Spec) SetDefaults() {
 }
 

parquet/spec_test.go

@@ -0,0 +1,25 @@
+package parquet
+
+import (
+	"testing"
+
+	"github.com/cloudquery/codegen/jsonschema"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSpec_JSONSchema(t *testing.T) {
+	schema, err := jsonschema.Generate(Spec{})
+	require.NoError(t, err)
+
+	jsonschema.TestJSONSchema(t, string(schema), []jsonschema.TestCase{
+		{
+			Name: "empty",
+			Spec: `{}`,
+		},
+		{
+			Name: "extra keyword",
+			Err:  true,
+			Spec: `{"extra":true}`,
+		},
+	})
+}