Add /contents endpoint to APIv2

api/v1/api.go

@@ -86,61 +86,61 @@ func (s *apiV1) RegisterRoutes(e *echo.Echo) {
 	e.POST("/register", s.handleRegisterUser)
 	e.POST("/login", s.handleLoginUser)
 	e.GET("/health", s.handleHealth)
-	e.GET("/viewer", withUser(s.handleGetViewer), s.AuthRequired(util.PermLevelUpload))
+	e.GET("/viewer", util.WithUser(s.handleGetViewer), s.AuthRequired(util.PermLevelUpload))
 	e.GET("/retrieval-candidates/:cid", s.handleGetRetrievalCandidates)
 	e.GET("/gw/:path", s.handleGateway)
 
-	e.POST("/put", util.WithMultipartFormDataChecker(withUser(s.handleAdd)), s.AuthRequired(util.PermLevelUpload))
+	e.POST("/put", util.WithMultipartFormDataChecker(util.WithUser(s.handleAdd)), s.AuthRequired(util.PermLevelUpload))
 	e.GET("/get/:cid", s.handleGetFullContentbyCid)
 	// e.HEAD("/get/:cid", s.handleGetContentByCid)
 
 	user := e.Group("/user")
 	user.Use(s.AuthRequired(util.PermLevelUser))
-	user.GET("/api-keys", withUser(s.handleUserGetApiKeys))
-	user.POST("/api-keys", withUser(s.handleUserCreateApiKey))
-	user.DELETE("/api-keys/:key_or_hash", withUser(s.handleUserRevokeApiKey))
-	user.GET("/export", withUser(s.handleUserExportData))
-	user.PUT("/password", withUser(s.handleUserChangePassword))
-	user.PUT("/address", withUser(s.handleUserChangeAddress))
-	user.GET("/stats", withUser(s.handleGetUserStats))
+	user.GET("/api-keys", util.WithUser(s.handleUserGetApiKeys))
+	user.POST("/api-keys", util.WithUser(s.handleUserCreateApiKey))
+	user.DELETE("/api-keys/:key_or_hash", util.WithUser(s.handleUserRevokeApiKey))
+	user.GET("/export", util.WithUser(s.handleUserExportData))
+	user.PUT("/password", util.WithUser(s.handleUserChangePassword))
+	user.PUT("/address", util.WithUser(s.handleUserChangeAddress))
+	user.GET("/stats", util.WithUser(s.handleGetUserStats))
 
 	userMiner := user.Group("/miner")
-	userMiner.POST("/claim", withUser(s.handleUserClaimMiner))
-	userMiner.GET("/claim/:miner", withUser(s.handleUserGetClaimMinerMsg))
-	userMiner.POST("/suspend/:miner", withUser(s.handleSuspendMiner))
-	userMiner.PUT("/unsuspend/:miner", withUser(s.handleUnsuspendMiner))
-	userMiner.PUT("/set-info/:miner", withUser(s.handleMinersSetInfo))
+	userMiner.POST("/claim", util.WithUser(s.handleUserClaimMiner))
+	userMiner.GET("/claim/:miner", util.WithUser(s.handleUserGetClaimMinerMsg))
+	userMiner.POST("/suspend/:miner", util.WithUser(s.handleSuspendMiner))
+	userMiner.PUT("/unsuspend/:miner", util.WithUser(s.handleUnsuspendMiner))
+	userMiner.PUT("/set-info/:miner", util.WithUser(s.handleMinersSetInfo))
 
 	contmeta := e.Group("/content")
 	uploads := contmeta.Group("", s.AuthRequired(util.PermLevelUpload))
-	uploads.POST("/add", util.WithMultipartFormDataChecker(withUser(s.handleAdd)))
-	uploads.POST("/add-ipfs", withUser(s.handleAddIpfs))
-	uploads.POST("/add-car", util.WithContentLengthCheck(withUser(s.handleAddCar)))
-	uploads.POST("/create", withUser(s.handleCreateContent))
+	uploads.POST("/add", util.WithMultipartFormDataChecker(util.WithUser(s.handleAdd)))
+	uploads.POST("/add-ipfs", util.WithUser(s.handleAddIpfs))
+	uploads.POST("/add-car", util.WithContentLengthCheck(util.WithUser(s.handleAddCar)))
+	uploads.POST("/create", util.WithUser(s.handleCreateContent))
 
 	content := contmeta.Group("", s.AuthRequired(util.PermLevelUser))
 	content.GET("/by-cid/:cid", s.handleGetContentByCid)
-	content.GET("/:cont_id", withUser(s.handleGetContent))
-	content.GET("/stats", withUser(s.handleStats))
+	content.GET("/:cont_id", util.WithUser(s.handleGetContent))
+	content.GET("/stats", util.WithUser(s.handleStats))
 	content.GET("/ensure-replication/:datacid", s.handleEnsureReplication)
-	content.GET("/status/:id", withUser(s.handleContentStatus))
-	content.GET("/list", withUser(s.handleListContent))
-	content.GET("/deals", withUser(s.handleListContentWithDeals))
-	content.GET("/failures/:content", withUser(s.handleGetContentFailures))
-	content.GET("/bw-usage/:content", withUser(s.handleGetContentBandwidth))
-	content.GET("/staging-zones", withUser(s.handleGetStagingZoneForUser))
-	content.GET("/aggregated/:content", withUser(s.handleGetAggregatedForContent))
-	content.GET("/all-deals", withUser(s.handleGetAllDealsForUser))
+	content.GET("/status/:id", util.WithUser(s.handleContentStatus))
+	content.GET("/list", util.WithUser(s.handleListContent))
+	content.GET("/deals", util.WithUser(s.handleListContentWithDeals))
+	content.GET("/failures/:content", util.WithUser(s.handleGetContentFailures))
+	content.GET("/bw-usage/:content", util.WithUser(s.handleGetContentBandwidth))
+	content.GET("/staging-zones", util.WithUser(s.handleGetStagingZoneForUser))
+	content.GET("/aggregated/:content", util.WithUser(s.handleGetAggregatedForContent))
+	content.GET("/all-deals", util.WithUser(s.handleGetAllDealsForUser))
 
 	// TODO: the commented out routes here are still fairly useful, but maybe
 	// need to have some sort of 'super user' permission level in order to use
 	// them? Can easily cause harm using them
 	deals := e.Group("/deals")
 	deals.Use(s.AuthRequired(util.PermLevelUser))
-	deals.GET("/status/:deal", withUser(s.handleGetDealStatus))
-	deals.GET("/status-by-proposal/:propcid", withUser(s.handleGetDealStatusByPropCid))
+	deals.GET("/status/:deal", util.WithUser(s.handleGetDealStatus))
+	deals.GET("/status-by-proposal/:propcid", util.WithUser(s.handleGetDealStatusByPropCid))
 	deals.GET("/query/:miner", s.handleQueryAsk)
-	deals.POST("/make/:miner", withUser(s.handleMakeDeal))
+	deals.POST("/make/:miner", util.WithUser(s.handleMakeDeal))
 	//deals.POST("/transfer/start/:miner/:propcid/:datacid", s.handleTransferStart)
 	deals.GET("/transfer/status/:id", s.handleTransferStatusByID)
 	deals.POST("/transfer/status", s.handleTransferStatus)
@@ -149,29 +149,29 @@ func (s *apiV1) RegisterRoutes(e *echo.Echo) {
 	deals.POST("/estimate", s.handleEstimateDealCost)
 	deals.GET("/proposal/:propcid", s.handleGetProposal)
 	deals.GET("/info/:dealid", s.handleGetDealInfo)
-	deals.GET("/failures", withUser(s.handleStorageFailures))
+	deals.GET("/failures", util.WithUser(s.handleStorageFailures))
 
 	cols := e.Group("/collections")
 	cols.Use(s.AuthRequired(util.PermLevelUser))
-	cols.GET("", withUser(s.handleListCollections))
-	cols.POST("", withUser(s.handleCreateCollection))
-	cols.DELETE("/:coluuid", withUser(s.handleDeleteCollection))
-	cols.POST("/:coluuid", withUser(s.handleAddContentsToCollection))
-	cols.GET("/:coluuid", withUser(s.handleGetCollectionContents))
-	cols.DELETE("/:coluuid/contents", withUser(s.handleDeleteContentFromCollection))
-	cols.POST("/:coluuid/commit", withUser(s.handleCommitCollection))
+	cols.GET("", util.WithUser(s.handleListCollections))
+	cols.POST("", util.WithUser(s.handleCreateCollection))
+	cols.DELETE("/:coluuid", util.WithUser(s.handleDeleteCollection))
+	cols.POST("/:coluuid", util.WithUser(s.handleAddContentsToCollection))
+	cols.GET("/:coluuid", util.WithUser(s.handleGetCollectionContents))
+	cols.DELETE("/:coluuid/contents", util.WithUser(s.handleDeleteContentFromCollection))
+	cols.POST("/:coluuid/commit", util.WithUser(s.handleCommitCollection))
 	colfs := cols.Group("/fs")
-	colfs.POST("/add", withUser(s.handleColfsAdd))
+	colfs.POST("/add", util.WithUser(s.handleColfsAdd))
 
 	pinning := e.Group("/pinning")
 	pinning.Use(util.OpenApiMiddleware(s.log))
 	pinning.Use(s.AuthRequired(util.PermLevelUser))
-	pinning.GET("/pins", withUser(s.handleListPins))
-	pinning.GET("/pins/:pinid", withUser(s.handleGetPin))
-	pinning.DELETE("/pins/:pinid", withUser(s.handleDeletePin))
+	pinning.GET("/pins", util.WithUser(s.handleListPins))
+	pinning.GET("/pins/:pinid", util.WithUser(s.handleGetPin))
+	pinning.DELETE("/pins/:pinid", util.WithUser(s.handleDeletePin))
 	pinning.Use(util.JSONPayloadMiddleware)
-	pinning.POST("/pins", withUser(s.handleAddPin))
-	pinning.POST("/pins/:pinid", withUser(s.handleReplacePin))
+	pinning.POST("/pins", util.WithUser(s.handleAddPin))
+	pinning.POST("/pins/:pinid", util.WithUser(s.handleReplacePin))
 
 	// explicitly public, for now
 	public := e.Group("/public")
@@ -204,14 +204,14 @@ func (s *apiV1) RegisterRoutes(e *echo.Echo) {
 	admin.GET("/dealstats", s.handleDealStats)
 	admin.GET("/disk-info", s.handleDiskSpaceCheck)
 	admin.GET("/stats", s.handleAdminStats)
-	admin.GET("/system/config", withUser(s.handleGetSystemConfig))
+	admin.GET("/system/config", util.WithUser(s.handleGetSystemConfig))
 
 	// miners
 	admin.POST("/miners/add/:miner", s.handleAdminAddMiner)
 	admin.POST("/miners/rm/:miner", s.handleAdminRemoveMiner)
-	admin.POST("/miners/suspend/:miner", withUser(s.handleSuspendMiner))
-	admin.PUT("/miners/unsuspend/:miner", withUser(s.handleUnsuspendMiner))
-	admin.PUT("/miners/set-info/:miner", withUser(s.handleMinersSetInfo))
+	admin.POST("/miners/suspend/:miner", util.WithUser(s.handleSuspendMiner))
+	admin.PUT("/miners/unsuspend/:miner", util.WithUser(s.handleUnsuspendMiner))
+	admin.PUT("/miners/set-info/:miner", util.WithUser(s.handleMinersSetInfo))
 	admin.GET("/miners", s.handleAdminGetMiners)
 	admin.GET("/miners/stats", s.handleAdminGetMinerStats)
 	admin.GET("/miners/transfers/:miner", s.handleMinerTransferDiagnostics)
@@ -248,7 +248,7 @@ func (s *apiV1) RegisterRoutes(e *echo.Echo) {
 	admin.GET("/retrieval/querytest/:content", s.handleRetrievalCheck)
 	admin.GET("/retrieval/stats", s.handleGetRetrievalInfo)
 
-	admin.POST("/invite/:code", withUser(s.handleAdminCreateInvite))
+	admin.POST("/invite/:code", util.WithUser(s.handleAdminCreateInvite))
 	admin.GET("/invites", s.handleAdminGetInvites)
 
 	admin.GET("/fixdeals", s.handleFixupDeals)

api/v1/handlers.go

@@ -107,20 +107,6 @@ type statsResp struct {
 	PinningStatus   pinningtypes.PinningStatus `json:"pinningStatus"`
 }
 
-func withUser(f func(echo.Context, *util.User) error) func(echo.Context) error {
-	return func(c echo.Context) error {
-		u, ok := c.Get("user").(*util.User)
-		if !ok {
-			return &util.HttpError{
-				Code:    http.StatusUnauthorized,
-				Reason:  util.ERR_INVALID_AUTH,
-				Details: "endpoint not called with proper authentication",
-			}
-		}
-		return f(c, u)
-	}
-}
-
 // handleStats godoc
 // @Summary      Get content statistics
 // @Description  This endpoint is used to get content statistics. Every content stored in the network (estuary) is tracked by a unique ID which can be used to get information about the content. This endpoint will allow the consumer to get the collected stats of a content
@@ -388,7 +374,7 @@ func (s *apiV1) handleAddIpfs(c echo.Context, u *util.User) error {
 		defaultPath := "/" + filename
 		colp := defaultPath
 		if params.CollectionDir != "" {
-			p, err := sanitizePath(params.CollectionDir)
+			p, err := util.SanitizePath(params.CollectionDir)
 			if err != nil {
 				return err
 			}
@@ -653,7 +639,7 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error {
 		col = &srchCol
 	}
 
-	path, err := constructDirectoryPath(c.QueryParam(ColDir))
+	path, err := util.ConstructDirectoryPath(c.QueryParam(ColDir))
 	if err != nil {
 		return err
 	}
@@ -735,20 +721,6 @@ func (s *apiV1) handleAdd(c echo.Context, u *util.User) error {
 	})
 }
 
-func constructDirectoryPath(dir string) (string, error) {
-	defaultPath := "/"
-	path := defaultPath
-	if cp := dir; cp != "" {
-		sp, err := sanitizePath(cp)
-		if err != nil {
-			return "", err
-		}
-
-		path = sp
-	}
-	return path, nil
-}
-
 // redirectContentAdding is called when localContentAddingDisabled is true
 // it finds available shuttles and adds the desired content in one of them
 func (s *apiV1) redirectContentAdding(c echo.Context, u *util.User) error {
@@ -3297,7 +3269,7 @@ func (s *apiV1) handleAddContentsToCollection(c echo.Context, u *util.User) erro
 		return fmt.Errorf("%d specified content(s) were not found or user missing permissions", len(contentIDs)-len(contents))
 	}
 
-	path, err := constructDirectoryPath(c.QueryParam(ColDir))
+	path, err := util.ConstructDirectoryPath(c.QueryParam(ColDir))
 	var colrefs []collections.CollectionRef
 	for _, cont := range contents {
 		fullPath := filepath.Join(path, cont.Name)
@@ -4659,7 +4631,7 @@ func (s *apiV1) handleCreateContent(c echo.Context, u *util.User) error {
 			req.CollectionDir = "/"
 		}
 
-		sp, err := sanitizePath(req.CollectionDir)
+		sp, err := util.SanitizePath(req.CollectionDir)
 		if err != nil {
 			return err
 		}
@@ -5062,25 +5034,15 @@ const (
 	ColDir string = "dir"
 )
 
-func sanitizePath(p string) (string, error) {
-	if len(p) == 0 {
-		return "", fmt.Errorf("can't sanitize empty path")
-	}
-
-	if p[0] != '/' {
-		return "", fmt.Errorf("paths must start with /")
-	}
-
-	// TODO: prevent use of special weird characters
-
-	cleanPath := filepath.Clean(p)
-
-	// if original path ends in /, append / to cleaned path
-	// needed for full path vs dir+filename magic to work in handleAddIpfs
-	if strings.HasSuffix(p, "/") {
-		cleanPath = cleanPath + "/"
-	}
-	return cleanPath, nil
+type collectionListResponse struct {
+	Name      string      `json:"name"`
+	Type      CidType     `json:"type"`
+	Size      int64       `json:"size"`
+	ContID    uint        `json:"contId"`
+	Cid       *util.DbCID `json:"cid,omitempty"`
+	Dir       string      `json:"dir"`
+	ColUuid   string      `json:"coluuid"`
+	UpdatedAt time.Time   `json:"updatedAt"`
 }
 
 // handleColfsAdd godoc
@@ -5134,7 +5096,7 @@ func (s *apiV1) handleColfsAdd(c echo.Context, u *util.User) error {
 
 	var path *string
 	if npath != "" {
-		p, err := sanitizePath(npath)
+		p, err := util.SanitizePath(npath)
 		if err != nil {
 			return err
 		}

api/v1/pinning.go

@@ -265,7 +265,7 @@ func (s *apiV1) handleAddPin(e echo.Context, u *util.User) error {
 		var colpath *string
 		colp, ok := pin.Meta["colpath"].(string)
 		if ok {
-			p, err := sanitizePath(colp)
+			p, err := util.SanitizePath(colp)
 			if err != nil {
 				return err
 			}

api/v2/api.go

@@ -7,6 +7,7 @@ import (
 	"github.com/application-research/estuary/node"
 	"github.com/application-research/estuary/pinner"
 	"github.com/application-research/estuary/stagingbs"
+	"github.com/application-research/estuary/util"
 	"github.com/application-research/estuary/util/gateway"
 	"github.com/application-research/filclient"
 	"github.com/filecoin-project/lotus/api"
@@ -19,13 +20,13 @@ import (
 
 type apiV2 struct {
 	cfg          *config.Estuary
-	DB           *gorm.DB
+	db           *gorm.DB
 	tracer       trace.Tracer
-	Node         *node.Node
-	FilClient    *filclient.FilClient
-	Api          api.Gateway
-	CM           *contentmgr.ContentManager
-	StagingMgr   *stagingbs.StagingBSMgr
+	node         *node.Node
+	filClient    *filclient.FilClient
+	api          api.Gateway
+	cm           *contentmgr.ContentManager
+	stagingMgr   *stagingbs.StagingBSMgr
 	gwayHandler  *gateway.GatewayHandler
 	cacher       *memo.Cacher
 	minerManager miner.IMinerManager
@@ -48,13 +49,13 @@ func NewAPIV2(
 ) *apiV2 {
 	return &apiV2{
 		cfg:          cfg,
-		DB:           db,
+		db:           db,
 		tracer:       trc,
-		Node:         nd,
-		FilClient:    fc,
-		Api:          gwApi,
-		CM:           cm,
-		StagingMgr:   sbm,
+		node:         nd,
+		filClient:    fc,
+		api:          gwApi,
+		cm:           cm,
+		stagingMgr:   sbm,
 		gwayHandler:  gateway.NewGatewayHandler(nd.Blockstore),
 		cacher:       memo.NewCacher(),
 		minerManager: mm,
@@ -81,5 +82,14 @@ func NewAPIV2(
 // @securityDefinitions.Bearer.in header
 // @securityDefinitions.Bearer.name Authorization
 func (s *apiV2) RegisterRoutes(e *echo.Echo) {
-	_ = e.Group("/v2")
+	e2 := e.Group("/v2")
+
+	// to upload contents you only need an upload key
+	// to see info about contents you need a user-level key (see contents group)
+	e2.POST("/contents", util.WithUser(s.handleAdd), s.AuthRequired(util.PermLevelUpload))
+	e2.GET("/contents", util.WithUser(s.handleListContent), s.AuthRequired(util.PermLevelUser))
+	e2.GET("/contents/:contentid", util.WithUser(s.handleGetContent), s.AuthRequired(util.PermLevelUser))
+	e2.GET("/contents/:cid/ensure-replication", s.handleEnsureReplication, s.AuthRequired(util.PermLevelUser))
+	e2.GET("/contents/:contentid/status", util.WithUser(s.handleContentStatus), s.AuthRequired(util.PermLevelUser))
+
 }