From 015067aad742c80c1894ba587f6598b5ecc2c85e Mon Sep 17 00:00:00 2001 From: yk-eukarya <81808708+yk-eukarya@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:05:58 +0100 Subject: [PATCH 1/2] fix(server): make the default value of number field clearable (#1289) --- server/pkg/value/number.go | 3 +++ server/pkg/value/number_test.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/server/pkg/value/number.go b/server/pkg/value/number.go index 3a346a6c65..50e3fd1d98 100644 --- a/server/pkg/value/number.go +++ b/server/pkg/value/number.go @@ -15,6 +15,9 @@ type propertyNumber struct{} type Number = float64 func (p *propertyNumber) ToValue(i any) (any, bool) { + if i == "" { + return nil, true + } switch v := i.(type) { case float64: return Number(v), true diff --git a/server/pkg/value/number_test.go b/server/pkg/value/number_test.go index b093ce0b03..9e862164a9 100644 --- a/server/pkg/value/number_test.go +++ b/server/pkg/value/number_test.go @@ -89,6 +89,12 @@ func Test_propertyNumber_ToValue(t *testing.T) { want1: float64(now.Unix()), want2: true, }, + { + name: "empty string", + args: []any{""}, + want1: nil, + want2: true, + }, { name: "nil", args: []any{"foo", (*float64)(nil), (*string)(nil), (*int)(nil), (*json.Number)(nil), nil}, From 3b29981c3cb35ab9d9fa26bc72870f1405ea42bb Mon Sep 17 00:00:00 2001 From: Nour Balaha Date: Tue, 29 Oct 2024 16:40:48 +0900 Subject: [PATCH 2/2] refactor(server): geojson and csv export (#1214) * refactor e2e tests * refactor csv in exporters * refactor * refactor ItemsWithProjectAsGeoJSON * test: TestCSVFromItems * fix GetFirstGeometryField * refactor BuildCSVHeaders --- server/e2e/integration_item_test.go | 23 +-- server/internal/adapter/integration/item.go | 22 +-- .../adapter/integration/item_export.go | 70 ++++++++ .../adapter/integration/item_export_test.go | 100 +++++++++++ server/internal/adapter/publicapi/csv.go | 39 ----- server/internal/adapter/publicapi/geojson.go | 118 ------------- .../internal/adapter/publicapi/item_export.go | 155 ++++++++++++++++++ server/pkg/exporters/csv.go | 80 ++------- server/pkg/exporters/csv_test.go | 131 +++------------ server/pkg/exporters/geojson.go | 41 +++-- server/pkg/exporters/geojson_test.go | 36 ++-- server/pkg/integrationapi/csv.go | 17 -- .../{geojson.go => item_export.go} | 44 +++-- server/pkg/item/item.go | 9 +- 14 files changed, 458 insertions(+), 427 deletions(-) create mode 100644 server/internal/adapter/integration/item_export.go create mode 100644 server/internal/adapter/integration/item_export_test.go delete mode 100644 server/internal/adapter/publicapi/csv.go delete mode 100644 server/internal/adapter/publicapi/geojson.go create mode 100644 server/internal/adapter/publicapi/item_export.go delete mode 100644 server/pkg/integrationapi/csv.go rename server/pkg/integrationapi/{geojson.go => item_export.go} (67%) diff --git a/server/e2e/integration_item_test.go b/server/e2e/integration_item_test.go index c83443a0a2..ef2a07bbc0 100644 --- a/server/e2e/integration_item_test.go +++ b/server/e2e/integration_item_test.go @@ -345,7 +345,8 @@ func baseSeeder(ctx context.Context, r *repo.Container) error { return err } - f := asset.NewFile().Name("aaa.jpg").Size(1000).ContentType("image/jpg").Build() + f1 := asset.NewFile().Name("aaa.jpg").Size(1000).ContentType("image/jpg").Build() + f2 := asset.NewFile().Name("bbb.jpg").Size(1000).ContentType("image/jpg").Build() a1 := asset.New().ID(aid1). Project(p.ID()). CreatedByUser(u.ID()). @@ -369,10 +370,10 @@ func baseSeeder(ctx context.Context, r *repo.Container) error { if err := r.Asset.Save(ctx, a2); err != nil { return err } - if err := r.AssetFile.Save(ctx, a1.ID(), f); err != nil { + if err := r.AssetFile.Save(ctx, a1.ID(), f1); err != nil { return err } - if err := r.AssetFile.Save(ctx, a2.ID(), f); err != nil { + if err := r.AssetFile.Save(ctx, a2.ID(), f2); err != nil { return err } // endregion @@ -443,7 +444,7 @@ func IntegrationSearchItem(e *httpexpect.Expect, mId string, page, perPage int, return res } -func IntegrationItemsAsGeoJSON(e *httpexpect.Expect, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.Value { +func IntegrationItemsAsGeoJSON(e *httpexpect.Expect, mId string, page, perPage int) *httpexpect.Value { res := e.GET("/api/models/{modelId}/items.geojson", mId). WithHeader("Origin", "https://example.com"). WithHeader("X-Reearth-Debug-User", uId1.String()). @@ -457,7 +458,7 @@ func IntegrationItemsAsGeoJSON(e *httpexpect.Expect, mId string, page, perPage i return res } -func IntegrationItemsWithProjectAsGeoJSON(e *httpexpect.Expect, pId string, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.Value { +func IntegrationItemsWithProjectAsGeoJSON(e *httpexpect.Expect, pId string, mId string, page, perPage int) *httpexpect.Value { res := e.GET("/api/projects/{projectIdOrAlias}/models/{modelIdOrKey}/items.geojson", pId, mId). WithHeader("Origin", "https://example.com"). WithHeader("X-Reearth-Debug-User", uId1.String()). @@ -471,7 +472,7 @@ func IntegrationItemsWithProjectAsGeoJSON(e *httpexpect.Expect, pId string, mId return res } -func IntegrationItemsAsCSV(e *httpexpect.Expect, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.String { +func IntegrationItemsAsCSV(e *httpexpect.Expect, mId string, page, perPage int) *httpexpect.String { res := e.GET("/api/models/{modelId}/items.csv", mId). WithHeader("Origin", "https://example.com"). WithHeader("X-Reearth-Debug-User", uId1.String()). @@ -485,7 +486,7 @@ func IntegrationItemsAsCSV(e *httpexpect.Expect, mId string, page, perPage int, return res } -func IntegrationItemsWithProjectAsCSV(e *httpexpect.Expect, pId string, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.String { +func IntegrationItemsWithProjectAsCSV(e *httpexpect.Expect, pId string, mId string, page, perPage int) *httpexpect.String { res := e.GET("/api/projects/{projectIdOrAlias}/models/{modelIdOrKey}/items.csv", pId, mId). WithHeader("Origin", "https://example.com"). WithHeader("X-Reearth-Debug-User", uId1.String()). @@ -1266,7 +1267,7 @@ func TestIntegrationItemsAsGeoJSON(t *testing.T) { {"schemaFieldId": fids.geometryObjectFid, "value": "{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}", "type": "GeometryObject"}, }) - res := IntegrationItemsAsGeoJSON(e, mId, 1, 10, i1Id, "", "", nil) + res := IntegrationItemsAsGeoJSON(e, mId, 1, 10) res.Object().Value("type").String().IsEqual("FeatureCollection") features := res.Object().Value("features").Array() features.Length().IsEqual(1) @@ -1294,7 +1295,7 @@ func TestIntegrationItemsWithProjectAsGeoJSON(t *testing.T) { {"schemaFieldId": fids.geometryEditorFid, "value": "{\"coordinates\": [[[138.90306434425662,36.11737907906834],[138.90306434425662,36.33622175736386],[138.67187898370287,36.33622175736386],[138.67187898370287,36.11737907906834],[138.90306434425662,36.11737907906834]]],\"type\": \"Polygon\"}", "type": "GeometryEditor"}, }) - res := IntegrationItemsWithProjectAsGeoJSON(e, pId, mId, 1, 10, i1Id, "", "", nil) + res := IntegrationItemsWithProjectAsGeoJSON(e, pId, mId, 1, 10) res.Object().Value("type").String().IsEqual("FeatureCollection") features := res.Object().Value("features").Array() features.Length().IsEqual(1) @@ -1321,7 +1322,7 @@ func TestIntegrationItemsAsCSV(t *testing.T) { {"schemaFieldId": fids.geometryObjectFid, "value": "{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}", "type": "GeometryObject"}, }) - res := IntegrationItemsAsCSV(e, mId, 1, 10, i1Id, "", "", nil) + res := IntegrationItemsAsCSV(e, mId, 1, 10) expected := fmt.Sprintf("id,location_lat,location_lng,text,textArea,markdown,asset,bool,select,integer,number,url,date,tag,checkbox\n%s,139.28179282584915,36.58570985749664,test1,,,,,,,,,,,\n", i1Id) res.IsEqual(expected) } @@ -1341,7 +1342,7 @@ func TestIntegrationItemsWithProjectAsCSV(t *testing.T) { {"schemaFieldId": fids.geometryEditorFid, "value": "{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}", "type": "GeometryEditor"}, }) - res := IntegrationItemsWithProjectAsCSV(e, pId, mId, 1, 10, i1Id, "", "", nil) + res := IntegrationItemsWithProjectAsCSV(e, pId, mId, 1, 10) expected := fmt.Sprintf("id,location_lat,location_lng,text,textArea,markdown,asset,bool,select,integer,number,url,date,tag,checkbox\n%s,139.28179282584915,36.58570985749664,test1,,,,,,30,,,,,\n", i1Id) res.IsEqual(expected) } diff --git a/server/internal/adapter/integration/item.go b/server/internal/adapter/integration/item.go index 94429266df..f79f876905 100644 --- a/server/internal/adapter/integration/item.go +++ b/server/internal/adapter/integration/item.go @@ -88,7 +88,7 @@ func (s *Server) ItemsAsGeoJSON(ctx context.Context, request ItemsAsGeoJSONReque return ItemsAsGeoJSON400Response{}, err } - fc, err := integrationapi.FeatureCollectionFromItems(items, sp.Schema()) + fc, err := featureCollectionFromItems(items, sp.Schema()) if err != nil { return ItemsAsGeoJSON400Response{}, err } @@ -118,10 +118,11 @@ func (s *Server) ItemsAsCSV(ctx context.Context, request ItemsAsCSVRequestObject } pr, pw := io.Pipe() - err = integrationapi.CSVFromItems(pw, items, sp.Schema()) + err = csvFromItems(pw, items, sp.Schema()) if err != nil { - return nil, err + return ItemsAsCSV400Response{}, err } + return ItemsAsCSV200TextcsvResponse{ Body: pr, }, nil @@ -203,9 +204,9 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit prj, err := uc.Project.FindByIDOrAlias(ctx, request.ProjectIdOrAlias, op) if err != nil { if errors.Is(err, rerror.ErrNotFound) { - return ItemsWithProjectAsGeoJSON400Response{}, err + return ItemsWithProjectAsGeoJSON404Response{}, err } - return nil, err + return ItemsWithProjectAsGeoJSON400Response{}, err } m, err := uc.Model.FindByIDOrKey(ctx, prj.ID(), request.ModelIdOrKey, op) @@ -230,7 +231,7 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit return ItemsWithProjectAsGeoJSON400Response{}, err } - fc, err := integrationapi.FeatureCollectionFromItems(items, sp.Schema()) + fc, err := featureCollectionFromItems(items, sp.Schema()) if err != nil { return ItemsWithProjectAsGeoJSON400Response{}, err } @@ -248,9 +249,9 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro prj, err := uc.Project.FindByIDOrAlias(ctx, request.ProjectIdOrAlias, op) if err != nil { if errors.Is(err, rerror.ErrNotFound) { - return ItemsWithProjectAsCSV400Response{}, err + return ItemsWithProjectAsCSV404Response{}, err } - return nil, err + return ItemsWithProjectAsCSV400Response{}, err } m, err := uc.Model.FindByIDOrKey(ctx, prj.ID(), request.ModelIdOrKey, op) @@ -276,10 +277,11 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro } pr, pw := io.Pipe() - err = integrationapi.CSVFromItems(pw, items, sp.Schema()) + err = csvFromItems(pw, items, sp.Schema()) if err != nil { - return nil, err + return ItemsWithProjectAsCSV400Response{}, err } + return ItemsWithProjectAsCSV200TextcsvResponse{ Body: pr, }, nil diff --git a/server/internal/adapter/integration/item_export.go b/server/internal/adapter/integration/item_export.go new file mode 100644 index 0000000000..fcc4ee77ee --- /dev/null +++ b/server/internal/adapter/integration/item_export.go @@ -0,0 +1,70 @@ +package integration + +import ( + "encoding/csv" + "io" + + "github.com/reearth/reearth-cms/server/pkg/integrationapi" + "github.com/reearth/reearth-cms/server/pkg/item" + "github.com/reearth/reearth-cms/server/pkg/schema" + "github.com/reearth/reearthx/i18n" + "github.com/reearth/reearthx/log" + "github.com/reearth/reearthx/rerror" + "github.com/samber/lo" +) + +var ( + pointFieldIsNotSupportedError = rerror.NewE(i18n.T("point type is not supported in any geometry field in this model")) +) + +// GeoJSON +func featureCollectionFromItems(ver item.VersionedList, s *schema.Schema) (*integrationapi.FeatureCollection, error) { + return integrationapi.FeatureCollectionFromItems(ver, s) +} + +// CSV +func csvFromItems(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { + if !s.IsPointFieldSupported() { + return pointFieldIsNotSupportedError + } + + go handleCSVGeneration(pw, l, s) + + return nil +} + +func handleCSVGeneration(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) { + err := generateCSV(pw, l, s) + if err != nil { + log.Errorf("failed to generate CSV: %v", err) + _ = pw.CloseWithError(err) + } else { + _ = pw.Close() + } +} + +func generateCSV(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { + w := csv.NewWriter(pw) + defer w.Flush() + + headers := integrationapi.BuildCSVHeaders(s) + if err := w.Write(headers); err != nil { + return err + } + + nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool { + return !f.IsGeometryField() + }) + + for _, ver := range l { + row, ok := integrationapi.RowFromItem(ver.Value(), nonGeoFields) + if ok { + if err := w.Write(row); err != nil { + return err + } + } + } + + return w.Error() +} + diff --git a/server/internal/adapter/integration/item_export_test.go b/server/internal/adapter/integration/item_export_test.go new file mode 100644 index 0000000000..774f31abf7 --- /dev/null +++ b/server/internal/adapter/integration/item_export_test.go @@ -0,0 +1,100 @@ +package integration + +import ( + "io" + "testing" + + "github.com/reearth/reearth-cms/server/pkg/id" + "github.com/reearth/reearth-cms/server/pkg/item" + "github.com/reearth/reearth-cms/server/pkg/schema" + "github.com/reearth/reearth-cms/server/pkg/value" + "github.com/reearth/reearth-cms/server/pkg/version" + "github.com/reearth/reearthx/account/accountdomain" + "github.com/reearth/reearthx/util" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" +) + +func TestCSVFromItems(t *testing.T) { + iid := id.NewItemID() + sid := id.NewSchemaID() + mid := id.NewModelID() + tid := id.NewThreadID() + pid := id.NewProjectID() + gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString} + gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString} + sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).MustBuild() + sf3 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).MustBuild() + in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100))) + tp4 := in4.TypeProperty() + sf4 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).MustBuild() + sf5 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("isMarried").Key(id.RandomKey()).MustBuild() + s1 := schema.New().ID(sid).Fields([]*schema.Field{sf1, sf3, sf4, sf5}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() + fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) + fi2 := item.NewField(sf3.ID(), value.TypeGeometryEditor.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) + fi3 := item.NewField(sf4.ID(), value.TypeInteger.Value(30).AsMultiple(), nil) + fi4 := item.NewField(sf5.ID(), value.TypeBool.Value(true).AsMultiple(), nil) + i1 := item.New(). + ID(iid). + Schema(sid). + Project(pid). + Fields([]*item.Field{fi1, fi2, fi3, fi4}). + Model(mid). + Thread(tid). + MustBuild() + v1 := version.New() + vi1 := version.MustBeValue(v1, nil, version.NewRefs(version.Latest), util.Now(), i1) + + // with geometry fields + ver1 := item.VersionedList{vi1} + _, pw := io.Pipe() + err := csvFromItems(pw, ver1, s1) + assert.Nil(t, err) + + // no geometry fields + iid2 := id.NewItemID() + sid2 := id.NewSchemaID() + mid2 := id.NewModelID() + tid2 := id.NewThreadID() + sf2 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() + s2 := schema.New().ID(sid).Fields([]*schema.Field{sf2}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() + i2 := item.New(). + ID(iid2). + Schema(sid2). + Project(pid). + Fields([]*item.Field{item.NewField(sf2.ID(), value.TypeText.Value("test").AsMultiple(), nil)}). + Model(mid2). + Thread(tid2). + MustBuild() + v2 := version.New() + vi2 := version.MustBeValue(v2, nil, version.NewRefs(version.Latest), util.Now(), i2) + ver2 := item.VersionedList{vi2} + expectErr2 := pointFieldIsNotSupportedError + _, pw1 := io.Pipe() + err = csvFromItems(pw1, ver2, s2) + assert.Equal(t, expectErr2, err) + + // point field is not supported + iid3 := id.NewItemID() + sid3 := id.NewSchemaID() + mid3 := id.NewModelID() + tid3 := id.NewThreadID() + gst2 := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypeLineString, schema.GeometryObjectSupportedTypePolygon} + sf6 := schema.NewField(schema.NewGeometryObject(gst2).TypeProperty()).NewID().Name("geo3").Key(id.RandomKey()).MustBuild() + s3 := schema.New().ID(sid).Fields([]*schema.Field{sf6}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() + i3 := item.New(). + ID(iid3). + Schema(sid3). + Project(pid). + Fields([]*item.Field{item.NewField(sf6.ID(), value.TypeText.Value("{\n \"coordinates\": [\n [\n 139.65439725962517,\n 36.34793305387103\n ],\n [\n 139.61688622815393,\n 35.910803456352724\n ]\n ],\n \"type\": \"LineString\"\n}").AsMultiple(), nil)}). + Model(mid3). + Thread(tid3). + MustBuild() + v3 := version.New() + vi3 := version.MustBeValue(v3, nil, version.NewRefs(version.Latest), util.Now(), i3) + ver3 := item.VersionedList{vi3} + expectErr3 := pointFieldIsNotSupportedError + _, pw2 := io.Pipe() + err = csvFromItems(pw2, ver3, s3) + assert.Equal(t, expectErr3, err) +} \ No newline at end of file diff --git a/server/internal/adapter/publicapi/csv.go b/server/internal/adapter/publicapi/csv.go deleted file mode 100644 index 291088a46f..0000000000 --- a/server/internal/adapter/publicapi/csv.go +++ /dev/null @@ -1,39 +0,0 @@ -package publicapi - -import ( - "io" - "net/http" - - "github.com/labstack/echo/v4" - "github.com/reearth/reearth-cms/server/pkg/exporters" - "github.com/reearth/reearth-cms/server/pkg/item" - "github.com/reearth/reearth-cms/server/pkg/schema" - "github.com/reearth/reearthx/log" -) - -func toCSV(c echo.Context, l item.VersionedList, s *schema.Schema) error { - if !s.IsPointFieldSupported() { - return c.JSON(http.StatusNotFound, map[string]interface{}{ - "error": "point type is not supported in this model", - }) - } - - pr, pw := io.Pipe() - err := generateCSV(pw, l, s) - if err != nil { - log.Errorf("failed to generate CSV: %+v", err) - } - - c.Response().Header().Set(echo.HeaderContentDisposition, "attachment;") - c.Response().Header().Set(echo.HeaderContentType, "text/csv") - return c.Stream(http.StatusOK, "text/csv", pr) -} - -func generateCSV(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { - err := exporters.CSVFromItems(pw, l, s) - if err != nil { - return err - } - - return nil -} diff --git a/server/internal/adapter/publicapi/geojson.go b/server/internal/adapter/publicapi/geojson.go deleted file mode 100644 index 79f155da0d..0000000000 --- a/server/internal/adapter/publicapi/geojson.go +++ /dev/null @@ -1,118 +0,0 @@ -package publicapi - -import ( - "encoding/json" - "io" - "net/http" - - "github.com/labstack/echo/v4" - "github.com/reearth/reearth-cms/server/pkg/exporters" - "github.com/reearth/reearth-cms/server/pkg/item" - "github.com/reearth/reearth-cms/server/pkg/schema" - "github.com/reearth/reearthx/log" - "github.com/samber/lo" -) - -func toGeoJSON(c echo.Context, l item.VersionedList, s *schema.Schema) error { - if !s.HasGeometryFields() { - return c.JSON(http.StatusNotFound, map[string]interface{}{ - "error": "no geometry field in this model", - }) - } - - pr, pw := io.Pipe() - go handleGeoJSONGeneration(pw, l, s) - - c.Response().Header().Set(echo.HeaderContentDisposition, "attachment;") - c.Response().Header().Set(echo.HeaderContentType, "application/json") - return c.Stream(http.StatusOK, "application/json", pr) -} - -func handleGeoJSONGeneration(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) { - err := generateGeoJSON(pw, l, s) - if err != nil { - log.Errorf("failed to generate GeoJSON: %+v", err) - } - _ = pw.CloseWithError(err) -} - -func generateGeoJSON(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { - features, err := exporters.FeatureCollectionFromItems(l, s) - - if err != nil { - return err - } - - featureCollection := ToFeatureCollection(features) - return json.NewEncoder(pw).Encode(featureCollection) -} - -func ToFeatureCollection(fc *exporters.FeatureCollection) *FeatureCollection { - if fc == nil || fc.Features == nil { - return nil - } - - features := lo.Map(*fc.Features, func(f exporters.Feature, _ int) Feature { - return *ToFeature(&f) - }) - - return &FeatureCollection{ - Type: lo.ToPtr(FeatureCollectionTypeFeatureCollection), - Features: &features, - } -} - -func ToFeature(f *exporters.Feature) *Feature { - if f == nil { - return nil - } - - return &Feature{ - Type: lo.ToPtr(FeatureTypeFeature), - Id: f.Id, - Geometry: ToGeometry(f.Geometry), - Properties: f.Properties, - } -} - -func ToGeometry(g *exporters.Geometry) *Geometry { - if g == nil { - return nil - } - - return &Geometry{ - Type: toGeometryType(*g.Type), - Coordinates: toCoordinates(*g.Coordinates), - } -} - -func toGeometryType(t exporters.GeometryType) *GeometryType { - switch t { - case exporters.GeometryTypePoint: - return lo.ToPtr(GeometryTypePoint) - case exporters.GeometryTypeMultiPoint: - return lo.ToPtr(GeometryTypeMultiPoint) - case exporters.GeometryTypeLineString: - return lo.ToPtr(GeometryTypeLineString) - case exporters.GeometryTypeMultiLineString: - return lo.ToPtr(GeometryTypeMultiLineString) - case exporters.GeometryTypePolygon: - return lo.ToPtr(GeometryTypePolygon) - case exporters.GeometryTypeMultiPolygon: - return lo.ToPtr(GeometryTypeMultiPolygon) - case exporters.GeometryTypeGeometryCollection: - return lo.ToPtr(GeometryTypeGeometryCollection) - default: - return nil - } -} - -func toCoordinates(c exporters.Geometry_Coordinates) *Geometry_Coordinates { - union, err := c.MarshalJSON() - if err != nil { - return nil - } - return &Geometry_Coordinates{ - union: union, - } -} diff --git a/server/internal/adapter/publicapi/item_export.go b/server/internal/adapter/publicapi/item_export.go new file mode 100644 index 0000000000..c9bf08c54a --- /dev/null +++ b/server/internal/adapter/publicapi/item_export.go @@ -0,0 +1,155 @@ +package publicapi + +import ( + "encoding/csv" + "io" + "net/http" + + "github.com/labstack/echo/v4" + "github.com/reearth/reearth-cms/server/pkg/exporters" + "github.com/reearth/reearth-cms/server/pkg/item" + "github.com/reearth/reearth-cms/server/pkg/schema" + "github.com/reearth/reearthx/log" + "github.com/samber/lo" +) + +// GeoJSON +func toGeoJSON(c echo.Context, l item.VersionedList, s *schema.Schema) error { + if !s.HasGeometryFields() { + return c.JSON(http.StatusBadRequest, map[string]interface{}{ + "error": "no geometry field in this model", + }) + } + + fc, err := exporters.FeatureCollectionFromItems(l, s) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "failed to generate GeoJSON").SetInternal(err) + } + + c.Response().Header().Set(echo.HeaderContentDisposition, "attachment;") + c.Response().Header().Set(echo.HeaderContentType, "application/json") + return c.JSON(http.StatusOK, toFeatureCollection(fc)) +} + +func toFeatureCollection(fc *exporters.FeatureCollection) *FeatureCollection { + if fc == nil || fc.Features == nil { + return nil + } + + features := lo.Map(*fc.Features, func(f exporters.Feature, _ int) Feature { + return toFeature(f) + }) + + return &FeatureCollection{ + Type: lo.ToPtr(FeatureCollectionTypeFeatureCollection), + Features: &features, + } +} + +func toFeature(f exporters.Feature) Feature { + return Feature{ + Type: lo.ToPtr(FeatureTypeFeature), + Id: f.Id, + Geometry: toGeometry(f.Geometry), + Properties: f.Properties, + } +} + +func toGeometry(g *exporters.Geometry) *Geometry { + if g == nil { + return nil + } + + return &Geometry{ + Type: toGeometryType(g.Type), + Coordinates: toCoordinates(g.Coordinates), + } +} + +func toGeometryType(t *exporters.GeometryType) *GeometryType { + if t == nil { + return nil + } + switch *t { + case exporters.GeometryTypePoint: + return lo.ToPtr(GeometryTypePoint) + case exporters.GeometryTypeMultiPoint: + return lo.ToPtr(GeometryTypeMultiPoint) + case exporters.GeometryTypeLineString: + return lo.ToPtr(GeometryTypeLineString) + case exporters.GeometryTypeMultiLineString: + return lo.ToPtr(GeometryTypeMultiLineString) + case exporters.GeometryTypePolygon: + return lo.ToPtr(GeometryTypePolygon) + case exporters.GeometryTypeMultiPolygon: + return lo.ToPtr(GeometryTypeMultiPolygon) + case exporters.GeometryTypeGeometryCollection: + return lo.ToPtr(GeometryTypeGeometryCollection) + default: + return nil + } +} + +func toCoordinates(c *exporters.Geometry_Coordinates) *Geometry_Coordinates { + if c == nil { + return nil + } + union, err := c.MarshalJSON() + if err != nil { + return nil + } + return &Geometry_Coordinates{ + union: union, + } +} + +// CSV +func toCSV(c echo.Context, l item.VersionedList, s *schema.Schema) error { + if !s.IsPointFieldSupported() { + return c.JSON(http.StatusBadRequest, map[string]string{ + "error": "point type is not supported in this model", + }) + } + + pr, pw := io.Pipe() + go handleCSVGeneration(pw, l, s) + + c.Response().Header().Set(echo.HeaderContentDisposition, "attachment;") + c.Response().Header().Set(echo.HeaderContentType, "text/csv") + return c.Stream(http.StatusOK, "text/csv", pr) +} + +func handleCSVGeneration(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) { + err := generateCSV(pw, l, s) + if err != nil { + log.Errorf("failed to generate CSV: %v", err) + _ = pw.CloseWithError(err) + } else { + _ = pw.Close() + } +} + +func generateCSV(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error { + w := csv.NewWriter(pw) + defer w.Flush() + + headers := exporters.BuildCSVHeaders(s) + if err := w.Write(headers); err != nil { + return err + } + + nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool { + return !f.IsGeometryField() + }) + + for _, ver := range l { + row, ok := exporters.RowFromItem(ver.Value(), nonGeoFields) + if ok { + if err := w.Write(row); err != nil { + return err + } + } + } + + return w.Error() +} \ No newline at end of file diff --git a/server/pkg/exporters/csv.go b/server/pkg/exporters/csv.go index d63b2b37d5..13e8e5c738 100644 --- a/server/pkg/exporters/csv.go +++ b/server/pkg/exporters/csv.go @@ -1,8 +1,6 @@ package exporters import ( - "encoding/csv" - "io" "strconv" "time" @@ -11,58 +9,23 @@ import ( "github.com/reearth/reearth-cms/server/pkg/value" "github.com/reearth/reearthx/i18n" "github.com/reearth/reearthx/rerror" - "github.com/samber/lo" ) var ( - noPointFieldError = rerror.NewE(i18n.T("no point field in this model")) - pointFieldIsNotSupportedError = rerror.NewE(i18n.T("point type is not supported in any geometry field in this model")) + noPointFieldError = rerror.NewE(i18n.T("no point field in this model")) ) -func CSVFromItems(pw *io.PipeWriter, items item.VersionedList, s *schema.Schema) error { - if !s.IsPointFieldSupported() { - return pointFieldIsNotSupportedError - } - - w := csv.NewWriter(pw) - go func() { - defer pw.Close() - - keys, nonGeoFields := buildCSVHeaders(s) - if err := w.Write(keys); err != nil { - pw.CloseWithError(err) - return - } - for _, ver := range items { - row, ok := rowFromItem(ver.Value(), nonGeoFields) - if ok { - if err := w.Write(row); err != nil { - pw.CloseWithError(err) - return - } - } - } - w.Flush() - if err := w.Error(); err != nil { - pw.CloseWithError(err) - } - }() - - return nil -} - -func buildCSVHeaders(s *schema.Schema) ([]string, []*schema.Field) { +func BuildCSVHeaders(s *schema.Schema) []string { keys := []string{"id", "location_lat", "location_lng"} - nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool { - return !f.IsGeometryField() - }) - for _, f := range nonGeoFields { - keys = append(keys, f.Name()) + for _, f := range s.Fields() { + if !f.IsGeometryField() { + keys = append(keys, f.Name()) + } } - return keys, nonGeoFields + return keys } -func rowFromItem(itm *item.Item, nonGeoFields []*schema.Field) ([]string, bool) { +func RowFromItem(itm *item.Item, nonGeoFields []*schema.Field) ([]string, bool) { geoField, err := extractFirstPointField(itm) if err != nil { return nil, false @@ -74,7 +37,7 @@ func rowFromItem(itm *item.Item, nonGeoFields []*schema.Field) ([]string, bool) for _, sf := range nonGeoFields { f := itm.Field(sf.ID()) - v := ToCSVProp(f) + v := toCSVProp(f) row = append(row, v) } @@ -82,20 +45,16 @@ func rowFromItem(itm *item.Item, nonGeoFields []*schema.Field) ([]string, bool) } func extractFirstPointField(itm *item.Item) ([]float64, error) { - geoFields := lo.Filter(itm.Fields(), func(f *item.Field, _ int) bool { - return f.Type().IsGeometryFieldType() - }) - - for _, f := range geoFields { - ss, ok := f.Value().First().ValueString() - if !ok { + for _, f := range itm.Fields() { + if !f.Type().IsGeometryFieldType() { continue } - g, err := StringToGeometry(ss) - if err != nil || g == nil { + ss, ok := f.Value().First().ValueString() + if !ok { continue } - if *g.Type != GeometryTypePoint { + g, err := stringToGeometry(ss) + if err != nil || g == nil || g.Type == nil || *g.Type != GeometryTypePoint { continue } return g.Coordinates.AsPoint() @@ -107,18 +66,15 @@ func float64ToString(f float64) string { return strconv.FormatFloat(f, 'f', -1, 64) } -func ToCSVProp(f *item.Field) string { +func toCSVProp(f *item.Field) string { if f == nil { return "" } vv := f.Value().First() - if vv == nil { - return "" - } - return ToCSVValue(vv) + return toCSVValue(vv) } -func ToCSVValue(vv *value.Value) string { +func toCSVValue(vv *value.Value) string { if vv == nil { return "" } diff --git a/server/pkg/exporters/csv_test.go b/server/pkg/exporters/csv_test.go index 5319543f56..0f102f7919 100644 --- a/server/pkg/exporters/csv_test.go +++ b/server/pkg/exporters/csv_test.go @@ -1,7 +1,6 @@ package exporters import ( - "io" "net/url" "testing" "time" @@ -10,97 +9,11 @@ import ( "github.com/reearth/reearth-cms/server/pkg/item" "github.com/reearth/reearth-cms/server/pkg/schema" "github.com/reearth/reearth-cms/server/pkg/value" - "github.com/reearth/reearth-cms/server/pkg/version" "github.com/reearth/reearthx/account/accountdomain" - "github.com/reearth/reearthx/util" "github.com/samber/lo" "github.com/stretchr/testify/assert" ) -func TestCSVFromItems(t *testing.T) { - iid := id.NewItemID() - sid := id.NewSchemaID() - mid := id.NewModelID() - tid := id.NewThreadID() - pid := id.NewProjectID() - gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString} - gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString} - sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).MustBuild() - sf3 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).MustBuild() - in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100))) - tp4 := in4.TypeProperty() - sf4 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).MustBuild() - sf5 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("isMarried").Key(id.RandomKey()).MustBuild() - s1 := schema.New().ID(sid).Fields([]*schema.Field{sf1, sf3, sf4, sf5}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() - fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) - fi2 := item.NewField(sf3.ID(), value.TypeGeometryEditor.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil) - fi3 := item.NewField(sf4.ID(), value.TypeInteger.Value(30).AsMultiple(), nil) - fi4 := item.NewField(sf5.ID(), value.TypeBool.Value(true).AsMultiple(), nil) - i1 := item.New(). - ID(iid). - Schema(sid). - Project(pid). - Fields([]*item.Field{fi1, fi2, fi3, fi4}). - Model(mid). - Thread(tid). - MustBuild() - v1 := version.New() - vi1 := version.MustBeValue(v1, nil, version.NewRefs(version.Latest), util.Now(), i1) - - // with geometry fields - ver1 := item.VersionedList{vi1} - _, pw := io.Pipe() - err := CSVFromItems(pw, ver1, s1) - assert.Nil(t, err) - - // no geometry fields - iid2 := id.NewItemID() - sid2 := id.NewSchemaID() - mid2 := id.NewModelID() - tid2 := id.NewThreadID() - sf2 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() - s2 := schema.New().ID(sid).Fields([]*schema.Field{sf2}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() - i2 := item.New(). - ID(iid2). - Schema(sid2). - Project(pid). - Fields([]*item.Field{item.NewField(sf2.ID(), value.TypeText.Value("test").AsMultiple(), nil)}). - Model(mid2). - Thread(tid2). - MustBuild() - v2 := version.New() - vi2 := version.MustBeValue(v2, nil, version.NewRefs(version.Latest), util.Now(), i2) - ver2 := item.VersionedList{vi2} - expectErr2 := pointFieldIsNotSupportedError - _, pw1 := io.Pipe() - err = CSVFromItems(pw1, ver2, s2) - assert.Equal(t, expectErr2, err) - - // point field is not supported - iid3 := id.NewItemID() - sid3 := id.NewSchemaID() - mid3 := id.NewModelID() - tid3 := id.NewThreadID() - gst2 := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypeLineString, schema.GeometryObjectSupportedTypePolygon} - sf6 := schema.NewField(schema.NewGeometryObject(gst2).TypeProperty()).NewID().Name("geo3").Key(id.RandomKey()).MustBuild() - s3 := schema.New().ID(sid).Fields([]*schema.Field{sf6}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() - i3 := item.New(). - ID(iid3). - Schema(sid3). - Project(pid). - Fields([]*item.Field{item.NewField(sf6.ID(), value.TypeText.Value("{\n \"coordinates\": [\n [\n 139.65439725962517,\n 36.34793305387103\n ],\n [\n 139.61688622815393,\n 35.910803456352724\n ]\n ],\n \"type\": \"LineString\"\n}").AsMultiple(), nil)}). - Model(mid3). - Thread(tid3). - MustBuild() - v3 := version.New() - vi3 := version.MustBeValue(v3, nil, version.NewRefs(version.Latest), util.Now(), i3) - ver3 := item.VersionedList{vi3} - expectErr3 := pointFieldIsNotSupportedError - _, pw2 := io.Pipe() - err = CSVFromItems(pw2, ver3, s3) - assert.Equal(t, expectErr3, err) -} - func TestBuildCSVHeaders(t *testing.T) { sid := id.NewSchemaID() pid := id.NewProjectID() @@ -116,14 +29,12 @@ func TestBuildCSVHeaders(t *testing.T) { s2 := schema.New().ID(sid).Fields([]*schema.Field{sf1, sf2, sf3, sf4}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild() // Test with geometry fields - headers1, ff := buildCSVHeaders(s1) + headers1 := BuildCSVHeaders(s1) assert.Equal(t, []string{"id", "location_lat", "location_lng", "age", "isMarried"}, headers1) - assert.Equal(t, []*schema.Field{sf3, sf4}, ff) // Test with mixed fields - headers2, _ := buildCSVHeaders(s2) + headers2 := BuildCSVHeaders(s2) assert.Equal(t, []string{"id", "location_lat", "location_lng", "age", "isMarried"}, headers2) - assert.Equal(t, []*schema.Field{sf3, sf4}, ff) } func TestRowFromItem(t *testing.T) { @@ -154,7 +65,7 @@ func TestRowFromItem(t *testing.T) { MustBuild() // Test with no fields - row1, ok1 := rowFromItem(i1, []*schema.Field{sf3, sf4}) + row1, ok1 := RowFromItem(i1, []*schema.Field{sf3, sf4}) assert.False(t, ok1) assert.Nil(t, row1) @@ -167,7 +78,7 @@ func TestRowFromItem(t *testing.T) { Model(mid). Thread(tid). MustBuild() - row2, ok2 := rowFromItem(i2, []*schema.Field{sf3, sf4}) + row2, ok2 := RowFromItem(i2, []*schema.Field{sf3, sf4}) assert.False(t, ok2) assert.Nil(t, row2) @@ -180,7 +91,7 @@ func TestRowFromItem(t *testing.T) { Model(mid). Thread(tid). MustBuild() - row3, ok3 := rowFromItem(i3, []*schema.Field{sf3, sf4}) + row3, ok3 := RowFromItem(i3, []*schema.Field{sf3, sf4}) assert.True(t, ok3) assert.Equal(t, []string{i1.ID().String(), "139.28179282584915", "36.58570985749664", "30", "true"}, row3) } @@ -249,11 +160,11 @@ func TestExtractFirstPointField(t *testing.T) { func TestToCSVProp(t *testing.T) { sf1 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if1 := item.NewField(sf1.ID(), value.TypeText.Value("test").AsMultiple(), nil) - s1 := ToCSVProp(if1) + s1 := toCSVProp(if1) assert.Equal(t, "test", s1) var if2 *item.Field - s2 := ToCSVProp(if2) + s2 := toCSVProp(if2) assert.Empty(t, s2) v3 := int64(30) @@ -261,21 +172,21 @@ func TestToCSVProp(t *testing.T) { tp3 := in3.TypeProperty() sf3 := schema.NewField(tp3).NewID().Name("age").Key(id.RandomKey()).MustBuild() if3 := item.NewField(sf3.ID(), value.TypeInteger.Value(v3).AsMultiple(), nil) - s3, ok3 := ToGeoJsonSingleValue(if3.Value().First()) + s3, ok3 := toGeoJsonSingleValue(if3.Value().First()) assert.Equal(t, int64(30), s3) assert.True(t, ok3) v4 := true sf4 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("age").Key(id.RandomKey()).MustBuild() if4 := item.NewField(sf4.ID(), value.TypeBool.Value(v4).AsMultiple(), nil) - s4, ok4 := ToGeoJsonSingleValue(if4.Value().First()) + s4, ok4 := toGeoJsonSingleValue(if4.Value().First()) assert.Equal(t, true, s4) assert.True(t, ok4) v5 := false sf5 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("age").Key(id.RandomKey()).MustBuild() if5 := item.NewField(sf5.ID(), value.TypeBool.Value(v5).AsMultiple(), nil) - s5, ok5 := ToGeoJsonSingleValue(if5.Value().First()) + s5, ok5 := toGeoJsonSingleValue(if5.Value().First()) assert.Equal(t, false, s5) assert.True(t, ok5) } @@ -283,36 +194,36 @@ func TestToCSVProp(t *testing.T) { func TestToCSVValue(t *testing.T) { sf1 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if1 := item.NewField(sf1.ID(), value.TypeText.Value("test").AsMultiple(), nil) - s1 := ToCSVValue(if1.Value().First()) + s1 := toCSVValue(if1.Value().First()) assert.Equal(t, "test", s1) sf2 := schema.NewField(schema.NewTextArea(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if2 := item.NewField(sf2.ID(), value.TypeTextArea.Value("test").AsMultiple(), nil) - s2 := ToCSVValue(if2.Value().First()) + s2 := toCSVValue(if2.Value().First()) assert.Equal(t, "test", s2) sf3 := schema.NewField(schema.NewURL().TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() v3 := url.URL{Scheme: "https", Host: "reearth.io"} if3 := item.NewField(sf3.ID(), value.TypeURL.Value(v3).AsMultiple(), nil) - s3 := ToCSVValue(if3.Value().First()) + s3 := toCSVValue(if3.Value().First()) assert.Equal(t, "https://reearth.io", s3) sf4 := schema.NewField(schema.NewAsset().TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if4 := item.NewField(sf4.ID(), value.TypeAsset.Value(id.NewAssetID()).AsMultiple(), nil) - s4 := ToCSVValue(if4.Value().First()) + s4 := toCSVValue(if4.Value().First()) assert.Empty(t, s4) gid := id.NewGroupID() igid := id.NewItemGroupID() sf5 := schema.NewField(schema.NewGroup(gid).TypeProperty()).NewID().Key(id.RandomKey()).Multiple(true).MustBuild() if5 := item.NewField(sf5.ID(), value.MultipleFrom(value.TypeGroup, []*value.Value{value.TypeGroup.Value(igid)}), nil) - s5 := ToCSVValue(if5.Value().First()) + s5 := toCSVValue(if5.Value().First()) assert.Empty(t, s5) v6 := id.NewItemID() sf6 := schema.NewField(schema.NewReference(id.NewModelID(), id.NewSchemaID(), nil, nil).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if6 := item.NewField(sf6.ID(), value.TypeReference.Value(v6).AsMultiple(), nil) - s6 := ToCSVValue(if6.Value().First()) + s6 := toCSVValue(if6.Value().First()) assert.Empty(t, s6) v7 := int64(30) @@ -320,7 +231,7 @@ func TestToCSVValue(t *testing.T) { tp7 := in7.TypeProperty() sf7 := schema.NewField(tp7).NewID().Name("age").Key(id.RandomKey()).MustBuild() if7 := item.NewField(sf7.ID(), value.TypeInteger.Value(v7).AsMultiple(), nil) - s7 := ToCSVValue(if7.Value().First()) + s7 := toCSVValue(if7.Value().First()) assert.Equal(t, "30", s7) v8 := float64(30.123) @@ -328,22 +239,22 @@ func TestToCSVValue(t *testing.T) { tp8 := in8.TypeProperty() sf8 := schema.NewField(tp8).NewID().Name("age").Key(id.RandomKey()).MustBuild() if8 := item.NewField(sf8.ID(), value.TypeNumber.Value(v8).AsMultiple(), nil) - s8 := ToCSVValue(if8.Value().First()) + s8 := toCSVValue(if8.Value().First()) assert.Equal(t, "30.123", s8) v9 := true sf9 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("age").Key(id.RandomKey()).MustBuild() if9 := item.NewField(sf9.ID(), value.TypeBool.Value(v9).AsMultiple(), nil) - s9 := ToCSVValue(if9.Value().First()) + s9 := toCSVValue(if9.Value().First()) assert.Equal(t, "true", s9) v10 := time.Now() sf10 := schema.NewField(schema.NewDateTime().TypeProperty()).NewID().Name("age").Key(id.RandomKey()).MustBuild() if10 := item.NewField(sf10.ID(), value.TypeDateTime.Value(v10).AsMultiple(), nil) - s10 := ToCSVValue(if10.Value().First()) + s10 := toCSVValue(if10.Value().First()) assert.Equal(t, v10.Format(time.RFC3339), s10) var if11 *item.Field - s11 := ToCSVValue(if11.Value().First()) + s11 := toCSVValue(if11.Value().First()) assert.Empty(t, s11) } diff --git a/server/pkg/exporters/geojson.go b/server/pkg/exporters/geojson.go index 2cbdc91787..18c121f890 100644 --- a/server/pkg/exporters/geojson.go +++ b/server/pkg/exporters/geojson.go @@ -149,7 +149,7 @@ func FeatureFromItem(ver item.Versioned, s *schema.Schema) (Feature, bool) { if !ok { return Feature{}, false } - geometry, ok := ExtractGeometry(geoField) + geometry, ok := extractGeometry(geoField) if !ok { return Feature{}, false } @@ -167,33 +167,31 @@ func extractProperties(itm *item.Item, s *schema.Schema) *map[string]any { return nil } properties := make(map[string]any) - nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool { - return f.Type() != value.TypeGeometryObject && f.Type() != value.TypeGeometryEditor - }) - for _, field := range nonGeoFields { - key := field.Name() - itmField := itm.Field(field.ID()) - val, ok := ToGeoJSONProp(itmField) - if ok { - properties[key] = val + for _, field := range s.Fields() { + if field.Type() != value.TypeGeometryObject && field.Type() != value.TypeGeometryEditor { + key := field.Name() + itmField := itm.Field(field.ID()) + if val, ok := toGeoJSONProp(itmField); ok { + properties[key] = val + } } } return &properties } -func ExtractGeometry(field *item.Field) (*Geometry, bool) { +func extractGeometry(field *item.Field) (*Geometry, bool) { geoStr, ok := field.Value().First().ValueString() if !ok { return nil, false } - geometry, err := StringToGeometry(geoStr) + geometry, err := stringToGeometry(geoStr) if err != nil { return nil, false } return geometry, true } -func StringToGeometry(geoString string) (*Geometry, error) { +func stringToGeometry(geoString string) (*Geometry, error) { var geometry Geometry if err := json.Unmarshal([]byte(geoString), &geometry); err != nil { return nil, err @@ -201,28 +199,27 @@ func StringToGeometry(geoString string) (*Geometry, error) { return &geometry, nil } -func ToGeoJSONProp(f *item.Field) (any, bool) { +func toGeoJSONProp(f *item.Field) (any, bool) { if f == nil { return nil, false } - if len(f.Value().Values()) == 1 { - return ToGeoJsonSingleValue(f.Value().First()) + if f.Value().Len() == 1 { + return toGeoJsonSingleValue(f.Value().First()) } - f.Type() m := value.MultipleFrom(f.Type(), f.Value().Values()) - return ToGeoJSONMultipleValues(m) + return toGeoJSONMultipleValues(m) } -func ToGeoJSONMultipleValues(m *value.Multiple) ([]any, bool) { - if len(m.Values()) == 0 { +func toGeoJSONMultipleValues(m *value.Multiple) ([]any, bool) { + if m.Len() == 0 { return nil, false } return lo.FilterMap(m.Values(), func(v *value.Value, _ int) (any, bool) { - return ToGeoJsonSingleValue(v) + return toGeoJsonSingleValue(v) }), true } -func ToGeoJsonSingleValue(vv *value.Value) (any, bool) { +func toGeoJsonSingleValue(vv *value.Value) (any, bool) { if vv == nil { return "", false } diff --git a/server/pkg/exporters/geojson_test.go b/server/pkg/exporters/geojson_test.go index e83fbce5fe..71ff88fa2b 100644 --- a/server/pkg/exporters/geojson_test.go +++ b/server/pkg/exporters/geojson_test.go @@ -119,13 +119,13 @@ func TestExtractGeometry(t *testing.T) { fi2 := item.NewField(sf2.ID(), value.MultipleFrom(value.TypeText, []*value.Value{value.TypeText.Value("a"), value.TypeText.Value("b"), value.TypeText.Value("c")}), nil) // Test with valid geometry field - geometry1, ok1 := ExtractGeometry(fi1) + geometry1, ok1 := extractGeometry(fi1) assert.True(t, ok1) assert.NotNil(t, geometry1) assert.Equal(t, GeometryTypeLineString, *geometry1.Type) // Test with non-geometry field - geometry2, ok2 := ExtractGeometry(fi2) + geometry2, ok2 := extractGeometry(fi2) assert.False(t, ok2) assert.Nil(t, geometry2) } @@ -205,7 +205,7 @@ func TestStringToGeometry(t *testing.T) { "type": "Point", "coordinates": [139.7112596, 35.6424892] }` - geo, err := StringToGeometry(validGeoStringPoint) + geo, err := stringToGeometry(validGeoStringPoint) assert.NoError(t, err) assert.NotNil(t, geo) assert.Equal(t, GeometryTypePoint, *geo.Type) @@ -216,7 +216,7 @@ func TestStringToGeometry(t *testing.T) { // Invalid Geometry string invalidGeometryString := "InvalidGeometry" - geo, err = StringToGeometry(invalidGeometryString) + geo, err = stringToGeometry(invalidGeometryString) assert.Error(t, err) assert.Nil(t, geo) } @@ -224,18 +224,18 @@ func TestStringToGeometry(t *testing.T) { func TestToGeoJSONProp(t *testing.T) { sf1 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if1 := item.NewField(sf1.ID(), value.TypeText.Value("test").AsMultiple(), nil) - s1, ok1 := ToGeoJSONProp(if1) + s1, ok1 := toGeoJSONProp(if1) assert.Equal(t, "test", s1) assert.True(t, ok1) var if2 *item.Field - s2, ok2 := ToGeoJSONProp(if2) + s2, ok2 := toGeoJSONProp(if2) assert.Empty(t, s2) assert.False(t, ok2) sf3 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if3 := item.NewField(sf3.ID(), value.MultipleFrom(value.TypeText, []*value.Value{value.TypeText.Value("a"), value.TypeText.Value("b"), value.TypeText.Value("c")}), nil) - s3, ok3 := ToGeoJSONProp(if3) + s3, ok3 := toGeoJSONProp(if3) assert.Equal(t, []any{"a", "b", "c"}, s3) assert.True(t, ok3) } @@ -243,26 +243,26 @@ func TestToGeoJSONProp(t *testing.T) { func TestToGeoJsonSingleValue(t *testing.T) { sf1 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if1 := item.NewField(sf1.ID(), value.TypeText.Value("test").AsMultiple(), nil) - s1, ok1 := ToGeoJsonSingleValue(if1.Value().First()) + s1, ok1 := toGeoJsonSingleValue(if1.Value().First()) assert.Equal(t, "test", s1) assert.True(t, ok1) sf2 := schema.NewField(schema.NewTextArea(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if2 := item.NewField(sf2.ID(), value.TypeTextArea.Value("test").AsMultiple(), nil) - s2, ok2 := ToGeoJsonSingleValue(if2.Value().First()) + s2, ok2 := toGeoJsonSingleValue(if2.Value().First()) assert.Equal(t, "test", s2) assert.True(t, ok2) sf3 := schema.NewField(schema.NewURL().TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() v3 := url.URL{Scheme: "https", Host: "reearth.io"} if3 := item.NewField(sf3.ID(), value.TypeURL.Value(v3).AsMultiple(), nil) - s3, ok3 := ToGeoJsonSingleValue(if3.Value().First()) + s3, ok3 := toGeoJsonSingleValue(if3.Value().First()) assert.Equal(t, "https://reearth.io", s3) assert.True(t, ok3) sf4 := schema.NewField(schema.NewAsset().TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if4 := item.NewField(sf4.ID(), value.TypeAsset.Value(id.NewAssetID()).AsMultiple(), nil) - s4, ok4 := ToGeoJsonSingleValue(if4.Value().First()) + s4, ok4 := toGeoJsonSingleValue(if4.Value().First()) assert.Empty(t, s4) assert.False(t, ok4) @@ -270,14 +270,14 @@ func TestToGeoJsonSingleValue(t *testing.T) { igid := id.NewItemGroupID() sf5 := schema.NewField(schema.NewGroup(gid).TypeProperty()).NewID().Key(id.RandomKey()).Multiple(true).MustBuild() if5 := item.NewField(sf5.ID(), value.MultipleFrom(value.TypeGroup, []*value.Value{value.TypeGroup.Value(igid)}), nil) - s5, ok5 := ToGeoJsonSingleValue(if5.Value().First()) + s5, ok5 := toGeoJsonSingleValue(if5.Value().First()) assert.Empty(t, s5) assert.False(t, ok5) v6 := id.NewItemID() sf6 := schema.NewField(schema.NewReference(id.NewModelID(), id.NewSchemaID(), nil, nil).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild() if6 := item.NewField(sf6.ID(), value.TypeReference.Value(v6).AsMultiple(), nil) - s6, ok6 := ToGeoJsonSingleValue(if6.Value().First()) + s6, ok6 := toGeoJsonSingleValue(if6.Value().First()) assert.Empty(t, s6) assert.False(t, ok6) @@ -286,7 +286,7 @@ func TestToGeoJsonSingleValue(t *testing.T) { tp7 := in7.TypeProperty() sf7 := schema.NewField(tp7).NewID().Name("age").Key(id.RandomKey()).MustBuild() if7 := item.NewField(sf7.ID(), value.TypeInteger.Value(v7).AsMultiple(), nil) - s7, ok7 := ToGeoJsonSingleValue(if7.Value().First()) + s7, ok7 := toGeoJsonSingleValue(if7.Value().First()) assert.Equal(t, int64(30), s7) assert.True(t, ok7) @@ -295,26 +295,26 @@ func TestToGeoJsonSingleValue(t *testing.T) { tp8 := in8.TypeProperty() sf8 := schema.NewField(tp8).NewID().Name("age").Key(id.RandomKey()).MustBuild() if8 := item.NewField(sf8.ID(), value.TypeNumber.Value(v8).AsMultiple(), nil) - s8, ok8 := ToGeoJsonSingleValue(if8.Value().First()) + s8, ok8 := toGeoJsonSingleValue(if8.Value().First()) assert.Equal(t, 30.123, s8) assert.True(t, ok8) v9 := true sf9 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("age").Key(id.RandomKey()).MustBuild() if9 := item.NewField(sf9.ID(), value.TypeBool.Value(v9).AsMultiple(), nil) - s9, ok9 := ToGeoJsonSingleValue(if9.Value().First()) + s9, ok9 := toGeoJsonSingleValue(if9.Value().First()) assert.Equal(t, true, s9) assert.True(t, ok9) v10 := time.Now() sf10 := schema.NewField(schema.NewDateTime().TypeProperty()).NewID().Name("age").Key(id.RandomKey()).MustBuild() if10 := item.NewField(sf10.ID(), value.TypeDateTime.Value(v10).AsMultiple(), nil) - s10, ok10 := ToGeoJsonSingleValue(if10.Value().First()) + s10, ok10 := toGeoJsonSingleValue(if10.Value().First()) assert.Equal(t, v10.Format(time.RFC3339), s10) assert.True(t, ok10) var if11 *item.Field - s11, ok11 := ToGeoJsonSingleValue(if11.Value().First()) + s11, ok11 := toGeoJsonSingleValue(if11.Value().First()) assert.Empty(t, s11) assert.False(t, ok11) } diff --git a/server/pkg/integrationapi/csv.go b/server/pkg/integrationapi/csv.go deleted file mode 100644 index 242d959806..0000000000 --- a/server/pkg/integrationapi/csv.go +++ /dev/null @@ -1,17 +0,0 @@ -package integrationapi - -import ( - "io" - - "github.com/reearth/reearth-cms/server/pkg/exporters" - "github.com/reearth/reearth-cms/server/pkg/item" - "github.com/reearth/reearth-cms/server/pkg/schema" -) - -func CSVFromItems(pw *io.PipeWriter, items item.VersionedList, s *schema.Schema) error { - err := exporters.CSVFromItems(pw, items, s) - if err != nil { - return err - } - return nil -} diff --git a/server/pkg/integrationapi/geojson.go b/server/pkg/integrationapi/item_export.go similarity index 67% rename from server/pkg/integrationapi/geojson.go rename to server/pkg/integrationapi/item_export.go index 21705a70f8..7015f6fb7a 100644 --- a/server/pkg/integrationapi/geojson.go +++ b/server/pkg/integrationapi/item_export.go @@ -8,21 +8,22 @@ import ( "github.com/samber/lo" ) +// GeoJSON func FeatureCollectionFromItems(ver item.VersionedList, s *schema.Schema) (*FeatureCollection, error) { fc, err := exporters.FeatureCollectionFromItems(ver, s) if err != nil { return nil, err } - return NewFeatureCollection(fc), nil + return newFeatureCollection(fc), nil } -func NewFeatureCollection(fc *exporters.FeatureCollection) *FeatureCollection { +func newFeatureCollection(fc *exporters.FeatureCollection) *FeatureCollection { if fc == nil || fc.Features == nil { return nil } features := lo.Map(*fc.Features, func(f exporters.Feature, _ int) Feature { - return *NewFeature(&f) + return newFeature(f) }) return &FeatureCollection{ @@ -31,32 +32,31 @@ func NewFeatureCollection(fc *exporters.FeatureCollection) *FeatureCollection { } } -func NewFeature(f *exporters.Feature) *Feature { - if f == nil { - return nil - } - - return &Feature{ +func newFeature(f exporters.Feature) Feature { + return Feature{ Type: lo.ToPtr(FeatureTypeFeature), Id: id.ItemIDFromRef(f.Id), - Geometry: NewGeometry(f.Geometry), + Geometry: newGeometry(f.Geometry), Properties: f.Properties, } } -func NewGeometry(g *exporters.Geometry) *Geometry { +func newGeometry(g *exporters.Geometry) *Geometry { if g == nil { return nil } return &Geometry{ - Type: toGeometryType(*g.Type), - Coordinates: toCoordinates(*g.Coordinates), + Type: toGeometryType(g.Type), + Coordinates: toCoordinates(g.Coordinates), } } -func toGeometryType(t exporters.GeometryType) *GeometryType { - switch t { +func toGeometryType(t *exporters.GeometryType) *GeometryType { + if t == nil { + return nil + } + switch *t { case exporters.GeometryTypePoint: return lo.ToPtr(GeometryTypePoint) case exporters.GeometryTypeMultiPoint: @@ -76,7 +76,10 @@ func toGeometryType(t exporters.GeometryType) *GeometryType { } } -func toCoordinates(c exporters.Geometry_Coordinates) *Geometry_Coordinates { +func toCoordinates(c *exporters.Geometry_Coordinates) *Geometry_Coordinates { + if c == nil { + return nil + } union, err := c.MarshalJSON() if err != nil { return nil @@ -85,3 +88,12 @@ func toCoordinates(c exporters.Geometry_Coordinates) *Geometry_Coordinates { union: union, } } + +// CSV +func BuildCSVHeaders(s *schema.Schema) []string { + return exporters.BuildCSVHeaders(s) +} + +func RowFromItem(itm *item.Item, nonGeoFields []*schema.Field) ([]string, bool) { + return exporters.RowFromItem(itm, nonGeoFields) +} diff --git a/server/pkg/item/item.go b/server/pkg/item/item.go index 7dde8d18b5..085e2e69da 100644 --- a/server/pkg/item/item.go +++ b/server/pkg/item/item.go @@ -260,9 +260,10 @@ func (i *Item) GetFirstGeometryField() (*Field, bool) { if i == nil { return nil, false } - geoFields := append(i.Fields().FieldsByType(value.TypeGeometryObject), i.Fields().FieldsByType(value.TypeGeometryEditor)...) - if len(geoFields) == 0 { - return nil, false + for _, f := range i.Fields() { + if f.IsGeometryField() && !f.Value().IsEmpty() { + return f, true + } } - return geoFields[0], true + return nil, false }