Skip to content

Commit

Permalink
Merge branch 'main' into fix-web/remove-member
Browse files Browse the repository at this point in the history
  • Loading branch information
caichi-t authored Oct 29, 2024
2 parents 36f68b2 + 3b29981 commit 18782c9
Show file tree
Hide file tree
Showing 16 changed files with 467 additions and 427 deletions.
23 changes: 12 additions & 11 deletions server/e2e/integration_item_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,8 @@ func baseSeeder(ctx context.Context, r *repo.Container) error {
return err
}

f := asset.NewFile().Name("aaa.jpg").Size(1000).ContentType("image/jpg").Build()
f1 := asset.NewFile().Name("aaa.jpg").Size(1000).ContentType("image/jpg").Build()
f2 := asset.NewFile().Name("bbb.jpg").Size(1000).ContentType("image/jpg").Build()
a1 := asset.New().ID(aid1).
Project(p.ID()).
CreatedByUser(u.ID()).
Expand All @@ -369,10 +370,10 @@ func baseSeeder(ctx context.Context, r *repo.Container) error {
if err := r.Asset.Save(ctx, a2); err != nil {
return err
}
if err := r.AssetFile.Save(ctx, a1.ID(), f); err != nil {
if err := r.AssetFile.Save(ctx, a1.ID(), f1); err != nil {
return err
}
if err := r.AssetFile.Save(ctx, a2.ID(), f); err != nil {
if err := r.AssetFile.Save(ctx, a2.ID(), f2); err != nil {
return err
}
// endregion
Expand Down Expand Up @@ -443,7 +444,7 @@ func IntegrationSearchItem(e *httpexpect.Expect, mId string, page, perPage int,
return res
}

func IntegrationItemsAsGeoJSON(e *httpexpect.Expect, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.Value {
func IntegrationItemsAsGeoJSON(e *httpexpect.Expect, mId string, page, perPage int) *httpexpect.Value {
res := e.GET("/api/models/{modelId}/items.geojson", mId).
WithHeader("Origin", "https://example.com").
WithHeader("X-Reearth-Debug-User", uId1.String()).
Expand All @@ -457,7 +458,7 @@ func IntegrationItemsAsGeoJSON(e *httpexpect.Expect, mId string, page, perPage i
return res
}

func IntegrationItemsWithProjectAsGeoJSON(e *httpexpect.Expect, pId string, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.Value {
func IntegrationItemsWithProjectAsGeoJSON(e *httpexpect.Expect, pId string, mId string, page, perPage int) *httpexpect.Value {
res := e.GET("/api/projects/{projectIdOrAlias}/models/{modelIdOrKey}/items.geojson", pId, mId).
WithHeader("Origin", "https://example.com").
WithHeader("X-Reearth-Debug-User", uId1.String()).
Expand All @@ -471,7 +472,7 @@ func IntegrationItemsWithProjectAsGeoJSON(e *httpexpect.Expect, pId string, mId
return res
}

func IntegrationItemsAsCSV(e *httpexpect.Expect, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.String {
func IntegrationItemsAsCSV(e *httpexpect.Expect, mId string, page, perPage int) *httpexpect.String {
res := e.GET("/api/models/{modelId}/items.csv", mId).
WithHeader("Origin", "https://example.com").
WithHeader("X-Reearth-Debug-User", uId1.String()).
Expand All @@ -485,7 +486,7 @@ func IntegrationItemsAsCSV(e *httpexpect.Expect, mId string, page, perPage int,
return res
}

func IntegrationItemsWithProjectAsCSV(e *httpexpect.Expect, pId string, mId string, page, perPage int, query string, sort, sortDir string, filter map[string]any) *httpexpect.String {
func IntegrationItemsWithProjectAsCSV(e *httpexpect.Expect, pId string, mId string, page, perPage int) *httpexpect.String {
res := e.GET("/api/projects/{projectIdOrAlias}/models/{modelIdOrKey}/items.csv", pId, mId).
WithHeader("Origin", "https://example.com").
WithHeader("X-Reearth-Debug-User", uId1.String()).
Expand Down Expand Up @@ -1266,7 +1267,7 @@ func TestIntegrationItemsAsGeoJSON(t *testing.T) {
{"schemaFieldId": fids.geometryObjectFid, "value": "{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}", "type": "GeometryObject"},
})

res := IntegrationItemsAsGeoJSON(e, mId, 1, 10, i1Id, "", "", nil)
res := IntegrationItemsAsGeoJSON(e, mId, 1, 10)
res.Object().Value("type").String().IsEqual("FeatureCollection")
features := res.Object().Value("features").Array()
features.Length().IsEqual(1)
Expand Down Expand Up @@ -1294,7 +1295,7 @@ func TestIntegrationItemsWithProjectAsGeoJSON(t *testing.T) {
{"schemaFieldId": fids.geometryEditorFid, "value": "{\"coordinates\": [[[138.90306434425662,36.11737907906834],[138.90306434425662,36.33622175736386],[138.67187898370287,36.33622175736386],[138.67187898370287,36.11737907906834],[138.90306434425662,36.11737907906834]]],\"type\": \"Polygon\"}", "type": "GeometryEditor"},
})

res := IntegrationItemsWithProjectAsGeoJSON(e, pId, mId, 1, 10, i1Id, "", "", nil)
res := IntegrationItemsWithProjectAsGeoJSON(e, pId, mId, 1, 10)
res.Object().Value("type").String().IsEqual("FeatureCollection")
features := res.Object().Value("features").Array()
features.Length().IsEqual(1)
Expand All @@ -1321,7 +1322,7 @@ func TestIntegrationItemsAsCSV(t *testing.T) {
{"schemaFieldId": fids.geometryObjectFid, "value": "{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}", "type": "GeometryObject"},
})

res := IntegrationItemsAsCSV(e, mId, 1, 10, i1Id, "", "", nil)
res := IntegrationItemsAsCSV(e, mId, 1, 10)
expected := fmt.Sprintf("id,location_lat,location_lng,text,textArea,markdown,asset,bool,select,integer,number,url,date,tag,checkbox\n%s,139.28179282584915,36.58570985749664,test1,,,,,,,,,,,\n", i1Id)
res.IsEqual(expected)
}
Expand All @@ -1341,7 +1342,7 @@ func TestIntegrationItemsWithProjectAsCSV(t *testing.T) {
{"schemaFieldId": fids.geometryEditorFid, "value": "{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}", "type": "GeometryEditor"},
})

res := IntegrationItemsWithProjectAsCSV(e, pId, mId, 1, 10, i1Id, "", "", nil)
res := IntegrationItemsWithProjectAsCSV(e, pId, mId, 1, 10)
expected := fmt.Sprintf("id,location_lat,location_lng,text,textArea,markdown,asset,bool,select,integer,number,url,date,tag,checkbox\n%s,139.28179282584915,36.58570985749664,test1,,,,,,30,,,,,\n", i1Id)
res.IsEqual(expected)
}
Expand Down
22 changes: 12 additions & 10 deletions server/internal/adapter/integration/item.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (s *Server) ItemsAsGeoJSON(ctx context.Context, request ItemsAsGeoJSONReque
return ItemsAsGeoJSON400Response{}, err
}

fc, err := integrationapi.FeatureCollectionFromItems(items, sp.Schema())
fc, err := featureCollectionFromItems(items, sp.Schema())
if err != nil {
return ItemsAsGeoJSON400Response{}, err
}
Expand Down Expand Up @@ -118,10 +118,11 @@ func (s *Server) ItemsAsCSV(ctx context.Context, request ItemsAsCSVRequestObject
}

pr, pw := io.Pipe()
err = integrationapi.CSVFromItems(pw, items, sp.Schema())
err = csvFromItems(pw, items, sp.Schema())
if err != nil {
return nil, err
return ItemsAsCSV400Response{}, err
}

return ItemsAsCSV200TextcsvResponse{
Body: pr,
}, nil
Expand Down Expand Up @@ -203,9 +204,9 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit
prj, err := uc.Project.FindByIDOrAlias(ctx, request.ProjectIdOrAlias, op)
if err != nil {
if errors.Is(err, rerror.ErrNotFound) {
return ItemsWithProjectAsGeoJSON400Response{}, err
return ItemsWithProjectAsGeoJSON404Response{}, err
}
return nil, err
return ItemsWithProjectAsGeoJSON400Response{}, err
}

m, err := uc.Model.FindByIDOrKey(ctx, prj.ID(), request.ModelIdOrKey, op)
Expand All @@ -230,7 +231,7 @@ func (s *Server) ItemsWithProjectAsGeoJSON(ctx context.Context, request ItemsWit
return ItemsWithProjectAsGeoJSON400Response{}, err
}

fc, err := integrationapi.FeatureCollectionFromItems(items, sp.Schema())
fc, err := featureCollectionFromItems(items, sp.Schema())
if err != nil {
return ItemsWithProjectAsGeoJSON400Response{}, err
}
Expand All @@ -248,9 +249,9 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro
prj, err := uc.Project.FindByIDOrAlias(ctx, request.ProjectIdOrAlias, op)
if err != nil {
if errors.Is(err, rerror.ErrNotFound) {
return ItemsWithProjectAsCSV400Response{}, err
return ItemsWithProjectAsCSV404Response{}, err
}
return nil, err
return ItemsWithProjectAsCSV400Response{}, err
}

m, err := uc.Model.FindByIDOrKey(ctx, prj.ID(), request.ModelIdOrKey, op)
Expand All @@ -276,10 +277,11 @@ func (s *Server) ItemsWithProjectAsCSV(ctx context.Context, request ItemsWithPro
}

pr, pw := io.Pipe()
err = integrationapi.CSVFromItems(pw, items, sp.Schema())
err = csvFromItems(pw, items, sp.Schema())
if err != nil {
return nil, err
return ItemsWithProjectAsCSV400Response{}, err
}

return ItemsWithProjectAsCSV200TextcsvResponse{
Body: pr,
}, nil
Expand Down
70 changes: 70 additions & 0 deletions server/internal/adapter/integration/item_export.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package integration

import (
"encoding/csv"
"io"

"github.com/reearth/reearth-cms/server/pkg/integrationapi"
"github.com/reearth/reearth-cms/server/pkg/item"
"github.com/reearth/reearth-cms/server/pkg/schema"
"github.com/reearth/reearthx/i18n"
"github.com/reearth/reearthx/log"
"github.com/reearth/reearthx/rerror"
"github.com/samber/lo"
)

var (
pointFieldIsNotSupportedError = rerror.NewE(i18n.T("point type is not supported in any geometry field in this model"))
)

// GeoJSON
func featureCollectionFromItems(ver item.VersionedList, s *schema.Schema) (*integrationapi.FeatureCollection, error) {
return integrationapi.FeatureCollectionFromItems(ver, s)
}

// CSV
func csvFromItems(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error {
if !s.IsPointFieldSupported() {
return pointFieldIsNotSupportedError
}

go handleCSVGeneration(pw, l, s)

return nil
}

func handleCSVGeneration(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) {
err := generateCSV(pw, l, s)
if err != nil {
log.Errorf("failed to generate CSV: %v", err)
_ = pw.CloseWithError(err)
} else {
_ = pw.Close()
}
}

func generateCSV(pw *io.PipeWriter, l item.VersionedList, s *schema.Schema) error {
w := csv.NewWriter(pw)
defer w.Flush()

headers := integrationapi.BuildCSVHeaders(s)
if err := w.Write(headers); err != nil {
return err
}

nonGeoFields := lo.Filter(s.Fields(), func(f *schema.Field, _ int) bool {
return !f.IsGeometryField()
})

for _, ver := range l {
row, ok := integrationapi.RowFromItem(ver.Value(), nonGeoFields)
if ok {
if err := w.Write(row); err != nil {
return err
}
}
}

return w.Error()
}

100 changes: 100 additions & 0 deletions server/internal/adapter/integration/item_export_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package integration

import (
"io"
"testing"

"github.com/reearth/reearth-cms/server/pkg/id"
"github.com/reearth/reearth-cms/server/pkg/item"
"github.com/reearth/reearth-cms/server/pkg/schema"
"github.com/reearth/reearth-cms/server/pkg/value"
"github.com/reearth/reearth-cms/server/pkg/version"
"github.com/reearth/reearthx/account/accountdomain"
"github.com/reearth/reearthx/util"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
)

func TestCSVFromItems(t *testing.T) {
iid := id.NewItemID()
sid := id.NewSchemaID()
mid := id.NewModelID()
tid := id.NewThreadID()
pid := id.NewProjectID()
gst := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypePoint, schema.GeometryObjectSupportedTypeLineString}
gest := schema.GeometryEditorSupportedTypeList{schema.GeometryEditorSupportedTypePoint, schema.GeometryEditorSupportedTypeLineString}
sf1 := schema.NewField(schema.NewGeometryObject(gst).TypeProperty()).NewID().Name("geo1").Key(id.RandomKey()).MustBuild()
sf3 := schema.NewField(schema.NewGeometryEditor(gest).TypeProperty()).NewID().Name("geo2").Key(id.RandomKey()).MustBuild()
in4, _ := schema.NewInteger(lo.ToPtr(int64(1)), lo.ToPtr(int64(100)))
tp4 := in4.TypeProperty()
sf4 := schema.NewField(tp4).NewID().Name("age").Key(id.RandomKey()).MustBuild()
sf5 := schema.NewField(schema.NewBool().TypeProperty()).NewID().Name("isMarried").Key(id.RandomKey()).MustBuild()
s1 := schema.New().ID(sid).Fields([]*schema.Field{sf1, sf3, sf4, sf5}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild()
fi1 := item.NewField(sf1.ID(), value.TypeGeometryObject.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil)
fi2 := item.NewField(sf3.ID(), value.TypeGeometryEditor.Value("{\"coordinates\":[139.28179282584915,36.58570985749664],\"type\":\"Point\"}").AsMultiple(), nil)
fi3 := item.NewField(sf4.ID(), value.TypeInteger.Value(30).AsMultiple(), nil)
fi4 := item.NewField(sf5.ID(), value.TypeBool.Value(true).AsMultiple(), nil)
i1 := item.New().
ID(iid).
Schema(sid).
Project(pid).
Fields([]*item.Field{fi1, fi2, fi3, fi4}).
Model(mid).
Thread(tid).
MustBuild()
v1 := version.New()
vi1 := version.MustBeValue(v1, nil, version.NewRefs(version.Latest), util.Now(), i1)

// with geometry fields
ver1 := item.VersionedList{vi1}
_, pw := io.Pipe()
err := csvFromItems(pw, ver1, s1)
assert.Nil(t, err)

// no geometry fields
iid2 := id.NewItemID()
sid2 := id.NewSchemaID()
mid2 := id.NewModelID()
tid2 := id.NewThreadID()
sf2 := schema.NewField(schema.NewText(lo.ToPtr(10)).TypeProperty()).NewID().Key(id.RandomKey()).MustBuild()
s2 := schema.New().ID(sid).Fields([]*schema.Field{sf2}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild()
i2 := item.New().
ID(iid2).
Schema(sid2).
Project(pid).
Fields([]*item.Field{item.NewField(sf2.ID(), value.TypeText.Value("test").AsMultiple(), nil)}).
Model(mid2).
Thread(tid2).
MustBuild()
v2 := version.New()
vi2 := version.MustBeValue(v2, nil, version.NewRefs(version.Latest), util.Now(), i2)
ver2 := item.VersionedList{vi2}
expectErr2 := pointFieldIsNotSupportedError
_, pw1 := io.Pipe()
err = csvFromItems(pw1, ver2, s2)
assert.Equal(t, expectErr2, err)

// point field is not supported
iid3 := id.NewItemID()
sid3 := id.NewSchemaID()
mid3 := id.NewModelID()
tid3 := id.NewThreadID()
gst2 := schema.GeometryObjectSupportedTypeList{schema.GeometryObjectSupportedTypeLineString, schema.GeometryObjectSupportedTypePolygon}
sf6 := schema.NewField(schema.NewGeometryObject(gst2).TypeProperty()).NewID().Name("geo3").Key(id.RandomKey()).MustBuild()
s3 := schema.New().ID(sid).Fields([]*schema.Field{sf6}).Workspace(accountdomain.NewWorkspaceID()).Project(pid).MustBuild()
i3 := item.New().
ID(iid3).
Schema(sid3).
Project(pid).
Fields([]*item.Field{item.NewField(sf6.ID(), value.TypeText.Value("{\n \"coordinates\": [\n [\n 139.65439725962517,\n 36.34793305387103\n ],\n [\n 139.61688622815393,\n 35.910803456352724\n ]\n ],\n \"type\": \"LineString\"\n}").AsMultiple(), nil)}).
Model(mid3).
Thread(tid3).
MustBuild()
v3 := version.New()
vi3 := version.MustBeValue(v3, nil, version.NewRefs(version.Latest), util.Now(), i3)
ver3 := item.VersionedList{vi3}
expectErr3 := pointFieldIsNotSupportedError
_, pw2 := io.Pipe()
err = csvFromItems(pw2, ver3, s3)
assert.Equal(t, expectErr3, err)
}
39 changes: 0 additions & 39 deletions server/internal/adapter/publicapi/csv.go

This file was deleted.

Loading

0 comments on commit 18782c9

Please sign in to comment.