add list project arifacts API (#20803)

* add list project arifacts API This API supports listing all artifacts belonging to a specified project. It also allows fetching the latest artifact in each repositry, with the option to filter by either media_type or artifact_type. Signed-off-by: wang yan <wangyan@vmware.com> * resolve the comments Signed-off-by: wang yan <wangyan@vmware.com> --------- Signed-off-by: wang yan <wangyan@vmware.com>
goharbor · Aug 6, 2024 · b7f1b59 · b7f1b59
1 parent 5deedf4
commit b7f1b59
Show file tree

Hide file tree

Showing 14 changed files with 462 additions and 10 deletions.
diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml
@@ -1548,6 +1548,88 @@ paths:
           $ref: '#/responses/409'
         '500':
           $ref: '#/responses/500'
+  /projects/{project_name_or_id}/artifacts:
+    get:
+      summary: List artifacts
+      description: List artifacts of the specified project
+      tags:
+        - project
+      operationId: listArtifactsOfProject
+      parameters:
+        - $ref: '#/parameters/requestId'
+        - $ref: '#/parameters/isResourceName'
+        - $ref: '#/parameters/projectNameOrId'
+        - $ref: '#/parameters/query'
+        - $ref: '#/parameters/sort'
+        - $ref: '#/parameters/page'
+        - $ref: '#/parameters/pageSize'
+        - $ref: '#/parameters/acceptVulnerabilities'
+        - name: with_tag
+          in: query
+          description: Specify whether the tags are included inside the returning artifacts
+          type: boolean
+          required: false
+          default: true
+        - name: with_label
+          in: query
+          description: Specify whether the labels are included inside the returning artifacts
+          type: boolean
+          required: false
+          default: false
+        - name: with_scan_overview
+          in: query
+          description: Specify whether the scan overview is included inside the returning artifacts
+          type: boolean
+          required: false
+          default: false
+        - name: with_sbom_overview
+          in: query
+          description: Specify whether the SBOM overview is included in returning artifacts, when this option is true, the SBOM overview will be included in the response
+          type: boolean
+          required: false
+          default: false
+        - name: with_immutable_status
+          in: query
+          description: Specify whether the immutable status is included inside the tags of the returning artifacts. Only works when setting "with_immutable_status=true"
+          type: boolean
+          required: false
+          default: false
+        - name: with_accessory
+          in: query
+          description: Specify whether the accessories are included of the returning artifacts. Only works when setting "with_accessory=true"
+          type: boolean
+          required: false
+          default: false
+        - name: latest_in_repository
+          in: query
+          description: Specify whether only the latest pushed artifact of each repository is included inside the returning artifacts. Only works when either artifact_type or media_type is included in the query.
+          type: boolean
+          required: false
+          default: false
+      responses:
+        '200':
+          description: Success
+          headers:
+            X-Total-Count:
+              description: The total count of artifacts
+              type: integer
+            Link:
+              description: Link refers to the previous page and next page
+              type: string
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/Artifact'
+        '400':
+          $ref: '#/responses/400'
+        '401':
+          $ref: '#/responses/401'
+        '403':
+          $ref: '#/responses/403'
+        '404':
+          $ref: '#/responses/404'
+        '500':
+          $ref: '#/responses/500'
   '/projects/{project_name_or_id}/scanner':
     get:
       summary: Get project level scanner
@@ -6586,6 +6668,9 @@ definitions:
       manifest_media_type:
         type: string
         description: The manifest media type of the artifact
+      artifact_type:
+        type: string
+        description: The artifact_type in the manifest of the artifact
       project_id:
         type: integer
         format: int64
@@ -6594,6 +6679,9 @@ definitions:
         type: integer
         format: int64
         description: The ID of the repository that the artifact belongs to
+      repository_name:
+        type: string
+        description: The name of the repository that the artifact belongs to
       digest:
         type: string
         description: The digest of the artifact

diff --git a/src/controller/artifact/controller.go b/src/controller/artifact/controller.go
@@ -118,6 +118,8 @@ type Controller interface {
 	Walk(ctx context.Context, root *Artifact, walkFn func(*Artifact) error, option *Option) error
 	// HasUnscannableLayer check artifact with digest if has unscannable layer
 	HasUnscannableLayer(ctx context.Context, dgst string) (bool, error)
+	// ListWithLatest list the artifacts when the latest_in_repository in the query was set
+	ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error)
 }
 
 // NewController creates an instance of the default artifact controller
@@ -782,3 +784,16 @@ func (c *controller) HasUnscannableLayer(ctx context.Context, dgst string) (bool
 	}
 	return false, nil
 }
+
+// ListWithLatest ...
+func (c *controller) ListWithLatest(ctx context.Context, query *q.Query, option *Option) (artifacts []*Artifact, err error) {
+	arts, err := c.artMgr.ListWithLatest(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	var res []*Artifact
+	for _, art := range arts {
+		res = append(res, c.assembleArtifact(ctx, art, option))
+	}
+	return res, nil
+}
diff --git a/src/controller/artifact/controller_test.go b/src/controller/artifact/controller_test.go
@@ -323,6 +323,44 @@ func (c *controllerTestSuite) TestList() {
 	c.Equal(0, len(artifacts[0].Accessories))
 }
 
+func (c *controllerTestSuite) TestListWithLatest() {
+	query := &q.Query{}
+	option := &Option{
+		WithTag:       true,
+		WithAccessory: true,
+	}
+	c.artMgr.On("ListWithLatest", mock.Anything, mock.Anything).Return([]*artifact.Artifact{
+		{
+			ID:           1,
+			RepositoryID: 1,
+		},
+	}, nil)
+	c.tagCtl.On("List").Return([]*tag.Tag{
+		{
+			Tag: model_tag.Tag{
+				ID:           1,
+				RepositoryID: 1,
+				ArtifactID:   1,
+				Name:         "latest",
+			},
+		},
+	}, nil)
+	c.repoMgr.On("Get", mock.Anything, mock.Anything).Return(&repomodel.RepoRecord{
+		Name: "library/hello-world",
+	}, nil)
+	c.repoMgr.On("List", mock.Anything, mock.Anything).Return([]*repomodel.RepoRecord{
+		{RepositoryID: 1, Name: "library/hello-world"},
+	}, nil)
+	c.accMgr.On("List", mock.Anything, mock.Anything).Return([]accessorymodel.Accessory{}, nil)
+	artifacts, err := c.ctl.ListWithLatest(nil, query, option)
+	c.Require().Nil(err)
+	c.Require().Len(artifacts, 1)
+	c.Equal(int64(1), artifacts[0].ID)
+	c.Require().Len(artifacts[0].Tags, 1)
+	c.Equal(int64(1), artifacts[0].Tags[0].ID)
+	c.Equal(0, len(artifacts[0].Accessories))
+}
+
 func (c *controllerTestSuite) TestGet() {
 	c.artMgr.On("Get", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{
 		ID:           1,

diff --git a/src/controller/artifact/model.go b/src/controller/artifact/model.go
@@ -102,8 +102,9 @@ type AdditionLink struct {
 
 // Option is used to specify the properties returned when listing/getting artifacts
 type Option struct {
-	WithTag       bool
-	TagOption     *tag.Option // only works when WithTag is set to true
-	WithLabel     bool
-	WithAccessory bool
+	WithTag            bool
+	TagOption          *tag.Option // only works when WithTag is set to true
+	WithLabel          bool
+	WithAccessory      bool
+	LatestInRepository bool
 }
diff --git a/src/pkg/artifact/dao/dao.go b/src/pkg/artifact/dao/dao.go
@@ -54,6 +54,8 @@ type DAO interface {
 	DeleteReference(ctx context.Context, id int64) (err error)
 	// DeleteReferences deletes the references referenced by the artifact specified by parent ID
 	DeleteReferences(ctx context.Context, parentID int64) (err error)
+	// ListWithLatest ...
+	ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error)
 }
 
 const (
@@ -282,6 +284,53 @@ func (d *dao) DeleteReferences(ctx context.Context, parentID int64) error {
 	return err
 }
 
+func (d *dao) ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error) {
+	ormer, err := orm.FromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	sql := `SELECT a.*
+	FROM artifact a
+	JOIN (
+		SELECT repository_name, MAX(push_time) AS latest_push_time
+	FROM artifact
+	WHERE project_id = ? and %s = ?
+	GROUP BY repository_name
+	) latest ON a.repository_name = latest.repository_name AND a.push_time = latest.latest_push_time`
+
+	queryParam := make([]interface{}, 0)
+	var ok bool
+	var pid interface{}
+	if pid, ok = query.Keywords["ProjectID"]; !ok {
+		return nil, errors.New(nil).WithCode(errors.BadRequestCode).
+			WithMessage(`the value of "ProjectID" must be set`)
+	}
+	queryParam = append(queryParam, pid)
+
+	var attributionValue interface{}
+	if attributionValue, ok = query.Keywords["media_type"]; ok {
+		sql = fmt.Sprintf(sql, "media_type")
+	} else if attributionValue, ok = query.Keywords["artifact_type"]; ok {
+		sql = fmt.Sprintf(sql, "artifact_type")
+	}
+
+	if attributionValue == "" {
+		return nil, errors.New(nil).WithCode(errors.BadRequestCode).
+			WithMessage(`the value of "media_type" or "artifact_type" must be set`)
+	}
+	queryParam = append(queryParam, attributionValue)
+
+	sql, queryParam = orm.PaginationOnRawSQL(query, sql, queryParam)
+	arts := []*Artifact{}
+	_, err = ormer.Raw(sql, queryParam...).QueryRows(&arts)
+	if err != nil {
+		return nil, err
+	}
+
+	return arts, nil
+}
+
 func querySetter(ctx context.Context, query *q.Query, options ...orm.Option) (beegoorm.QuerySeter, error) {
 	qs, err := orm.QuerySetter(ctx, &Artifact{}, query, options...)
 	if err != nil {

diff --git a/src/pkg/artifact/dao/dao_test.go b/src/pkg/artifact/dao/dao_test.go
@@ -472,6 +472,75 @@ func (d *daoTestSuite) TestDeleteReferences() {
 	d.True(errors.IsErr(err, errors.NotFoundCode))
 }
 
+func (d *daoTestSuite) TestListWithLatest() {
+	now := time.Now()
+	art := &Artifact{
+		Type:              "IMAGE",
+		MediaType:         v1.MediaTypeImageConfig,
+		ManifestMediaType: v1.MediaTypeImageIndex,
+		ProjectID:         1234,
+		RepositoryID:      1234,
+		RepositoryName:    "library2/hello-world1",
+		Digest:            "digest",
+		PushTime:          now,
+		PullTime:          now,
+		Annotations:       `{"anno1":"value1"}`,
+	}
+	id, err := d.dao.Create(d.ctx, art)
+	d.Require().Nil(err)
+
+	time.Sleep(1 * time.Second)
+	now = time.Now()
+
+	art2 := &Artifact{
+		Type:              "IMAGE",
+		MediaType:         v1.MediaTypeImageConfig,
+		ManifestMediaType: v1.MediaTypeImageIndex,
+		ProjectID:         1234,
+		RepositoryID:      1235,
+		RepositoryName:    "library2/hello-world2",
+		Digest:            "digest",
+		PushTime:          now,
+		PullTime:          now,
+		Annotations:       `{"anno1":"value1"}`,
+	}
+	id1, err := d.dao.Create(d.ctx, art2)
+	d.Require().Nil(err)
+
+	time.Sleep(1 * time.Second)
+	now = time.Now()
+
+	art3 := &Artifact{
+		Type:              "IMAGE",
+		MediaType:         v1.MediaTypeImageConfig,
+		ManifestMediaType: v1.MediaTypeImageIndex,
+		ProjectID:         1234,
+		RepositoryID:      1235,
+		RepositoryName:    "library2/hello-world2",
+		Digest:            "digest2",
+		PushTime:          now,
+		PullTime:          now,
+		Annotations:       `{"anno1":"value1"}`,
+	}
+	id2, err := d.dao.Create(d.ctx, art3)
+	d.Require().Nil(err)
+
+	latest, err := d.dao.ListWithLatest(d.ctx, &q.Query{
+		Keywords: map[string]interface{}{
+			"ProjectID":  1234,
+			"media_type": v1.MediaTypeImageConfig,
+		},
+	})
+
+	d.Require().Nil(err)
+	d.Require().Equal(2, len(latest))
+	d.Equal("library2/hello-world1", latest[0].RepositoryName)
+
+	defer d.dao.Delete(d.ctx, id)
+	defer d.dao.Delete(d.ctx, id1)
+	defer d.dao.Delete(d.ctx, id2)
+}
+
 func TestDaoTestSuite(t *testing.T) {
 	suite.Run(t, &daoTestSuite{})
 }
diff --git a/src/pkg/artifact/manager.go b/src/pkg/artifact/manager.go
@@ -48,6 +48,8 @@ type Manager interface {
 	ListReferences(ctx context.Context, query *q.Query) (references []*Reference, err error)
 	// DeleteReference specified by ID
 	DeleteReference(ctx context.Context, id int64) (err error)
+	// ListWithLatest list the artifacts when the latest_in_repository in the query was set
+	ListWithLatest(ctx context.Context, query *q.Query) (artifacts []*Artifact, err error)
 }
 
 // NewManager returns an instance of the default manager
@@ -147,6 +149,22 @@ func (m *manager) DeleteReference(ctx context.Context, id int64) error {
 	return m.dao.DeleteReference(ctx, id)
 }
 
+func (m *manager) ListWithLatest(ctx context.Context, query *q.Query) ([]*Artifact, error) {
+	arts, err := m.dao.ListWithLatest(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	var artifacts []*Artifact
+	for _, art := range arts {
+		artifact, err := m.assemble(ctx, art)
+		if err != nil {
+			return nil, err
+		}
+		artifacts = append(artifacts, artifact)
+	}
+	return artifacts, nil
+}
+
 // assemble the artifact with references populated
 func (m *manager) assemble(ctx context.Context, art *dao.Artifact) (*Artifact, error) {
 	artifact := &Artifact{}