Implement actions artifacts

models/actions/artifact.go

@@ -0,0 +1,126 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// This artifact server is inspired by https://github.com/nektos/act/blob/master/pkg/artifacts/server.go.
+// It updates url setting and uses ObjectStore to handle artifacts persistence.
+
+package actions
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
+)
+
+const (
+	// ArtifactUploadStatusPending is the status of an artifact upload that is pending
+	ArtifactUploadStatusPending = 1
+	// ArtifactUploadStatusConfirmed is the status of an artifact upload that is confirmed
+	ArtifactUploadStatusConfirmed = 2
+)
+
+func init() {
+	db.RegisterModel(new(ActionArtifact))
+}
+
+// ActionArtifact is a file that is stored in the artifact storage.
+type ActionArtifact struct {
+	ID               int64 `xorm:"pk autoincr"`
+	RunID            int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
+	RunnerID         int64
+	RepoID           int64 `xorm:"index"`
+	OwnerID          int64
+	CommitSHA        string
+	StoragePath      string             // The path to the artifact in the storage
+	FileSize         int64              // The size of the artifact in bytes
+	FileGzipSize     int64              // The size of the artifact in bytes after gzip compression
+	ContentEncnoding string             // The content encoding of the artifact
+	ArtifactPath     string             // The path to the artifact when runner uploads it
+	ArtifactName     string             `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
+	UploadStatus     int64              `xorm:"index"`              // The status of the artifact upload
+	Created          timeutil.TimeStamp `xorm:"created"`
+	Updated          timeutil.TimeStamp `xorm:"updated index"`
+}
+
+// CreateArtifact create a new artifact with task info or get same named artifact in the same run
+func CreateArtifact(ctx context.Context, t *ActionTask, artifactName string) (*ActionArtifact, error) {
+	if t.Job == nil {
+		if err := t.LoadJob(ctx); err != nil {
+			return nil, err
+		}
+	}
+	if err := t.Job.LoadRun(ctx); err != nil {
+		return nil, err
+	}
+	artifact, err := GetArtifactByArtifactName(ctx, t.Job.RunID, artifactName)
+	if err != nil {
+		return nil, err
+	}
+	if artifact != nil {
+		return artifact, nil
+	}
+	artifact = &ActionArtifact{
+		RunID:        t.Job.RunID,
+		RunnerID:     t.RunnerID,
+		RepoID:       t.RepoID,
+		OwnerID:      t.OwnerID,
+		CommitSHA:    t.CommitSHA,
+		UploadStatus: ArtifactUploadStatusPending,
+	}
+	if _, err := db.GetEngine(ctx).Insert(artifact); err != nil {
+		return nil, err
+	}
+	return artifact, nil
+}
+
+// GetArtifactByArtifactName returns an artifact by name
+func GetArtifactByArtifactName(ctx context.Context, runID int64, name string) (*ActionArtifact, error) {
+	var art ActionArtifact
+	has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ?", runID, name).Get(&art)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, nil
+	}
+	return &art, nil
+}
+
+// GetArtifactByID returns an artifact by id
+func GetArtifactByID(ctx context.Context, id int64) (*ActionArtifact, error) {
+	var art ActionArtifact
+	has, err := db.GetEngine(ctx).ID(id).Get(&art)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, util.NewNotExistErrorf("no ActionArtifact with id %d exists", id)
+	}
+
+	return &art, nil
+}
+
+// UpdateArtifactByID updates an artifact by id
+func UpdateArtifactByID(ctx context.Context, id int64, art *ActionArtifact) error {
+	art.ID = id
+	_, err := db.GetEngine(ctx).ID(id).AllCols().Update(art)
+	return err
+}
+
+// ListArtifactByRunID returns all artifacts of a run
+func ListArtifactByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).Where("run_id=?", runID).Find(&arts)
+}
+
+// ListUploadedArtifactByRunID returns all uploaded artifacts of a run
+func ListUploadedArtifactByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).Where("run_id=? AND upload_status=?", runID, ArtifactUploadStatusConfirmed).Find(&arts)
+}
+
+// ListArtifactsByRepoID returns all artifacts of a repo
+func ListArtifactsByRepoID(ctx context.Context, repoID int64) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&arts)
+}

models/fixtures/access_token.yml

@@ -30,4 +30,4 @@
   token_last_eight: 69d28c91
   created_unix: 946687980
   updated_unix: 946687980
-#commented out tokens so you can see what they are in plaintext
+#commented out tokens so you can see what they are in plaintext

models/fixtures/action_run.yml

@@ -0,0 +1,20 @@
+-
+  id: 791
+  title: "update actions"
+  repo_id: 4
+  owner_id: 1
+  workflow_id: "artifact.yaml"
+  index: 187
+  trigger_user_id: 1
+  ref: "refs/heads/master"
+  commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
+  event: "push"
+  is_fork_pull_request: 0
+  event_payload: ""
+  status: 1
+  started: 1683636528
+  stopped: 1683636626
+  created: 1683636108
+  updated: 1683636626
+  need_approval: 0
+  approved_by: 0

models/fixtures/action_run_job.yml

@@ -0,0 +1,19 @@
+-
+  id: 192
+  run_id: 791
+  repo_id: 4
+  owner_id: 1
+  commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
+  is_fork_pull_request: 0
+  name: "job_2"
+  attempt: 1
+  workflow_payload: ""
+  job_id: "job_2"
+  needs: ""
+  runs_on: "[\"ubuntu-latest\"]"
+  task_id: 47
+  status: 1
+  started: 1683636528
+  stopped: 1683636626
+  created: 1683636108
+  updated: 1683636626

models/fixtures/action_task.yml

@@ -0,0 +1,23 @@
+-
+  id: 47
+  job_id: 192
+  attempt: 3
+  runner_id: 1
+  status: 6 # 6 is the status code for "running", running task can upload artifacts
+  started: 1683636528
+  stopped: 1683636626
+  repo_id: 4
+  owner_id: 1
+  commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
+  is_fork_pull_request: 0
+  token_hash: "6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4260c64a69a2cc1508825121b7b8394e48e00b1bf8718b2a867e"
+  token_salt: "jVuKnSPGgy"
+  token_last_eight: "eeb1a71a"
+  log_filename: "artifact-test2/2f/47.log"
+  log_in_storage: 1
+  log_length: 707
+  log_size: 90179
+  log_indexes: ""
+  log_expired: 0
+  created: 1683636528
+  updated: 1683646528

models/migrations/migrations.go

@@ -489,6 +489,8 @@ var migrations = []Migration{
 	NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable),
 	// v255 -> v256
 	NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
+	// v256 -> v257
+	NewMigration("Add Actions Artifact table", v1_20.CreateActionArtifactTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_20/v256.go

@@ -0,0 +1,33 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_20 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreateActionArtifactTable(x *xorm.Engine) error {
+	// ActionArtifact is a file that is stored in the artifact storage.
+	type ActionArtifact struct {
+		ID               int64 `xorm:"pk autoincr"`
+		RunID            int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
+		RunnerID         int64
+		RepoID           int64 `xorm:"index"`
+		OwnerID          int64
+		CommitSHA        string
+		StoragePath      string             // The path to the artifact in the storage
+		FileSize         int64              // The size of the artifact in bytes
+		FileGzipSize     int64              // The size of the artifact in bytes after gzip compression
+		ContentEncnoding string             // The content encoding of the artifact
+		ArtifactPath     string             // The path to the artifact when runner uploads it
+		ArtifactName     string             `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
+		UploadStatus     int64              `xorm:"index"`              // The status of the artifact upload
+		Created          timeutil.TimeStamp `xorm:"created"`
+		Updated          timeutil.TimeStamp `xorm:"updated index"`
+	}
+
+	return x.Sync(new(ActionArtifact))
+}

models/repo.go

@@ -60,6 +60,12 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		return fmt.Errorf("find actions tasks of repo %v: %w", repoID, err)
 	}
 
+	// Query the artifacts of this repo, they will be needed after they have been deleted to remove artifacts files in ObjectStorage
+	artifacts, err := actions_model.ListArtifactsByRepoID(ctx, repoID)
+	if err != nil {
+		return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err)
+	}
+
 	// In case is a organization.
 	org, err := user_model.GetUserByID(ctx, uid)
 	if err != nil {
@@ -165,6 +171,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		&actions_model.ActionRunJob{RepoID: repoID},
 		&actions_model.ActionRun{RepoID: repoID},
 		&actions_model.ActionRunner{RepoID: repoID},
+		&actions_model.ActionArtifact{RepoID: repoID},
 	); err != nil {
 		return fmt.Errorf("deleteBeans: %w", err)
 	}
@@ -337,6 +344,14 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		}
 	}
 
+	// delete actions artifacts in ObjectStorage after the repo have already been deleted
+	for _, art := range artifacts {
+		if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil {
+			log.Error("remove artifact file %q: %v", art.StoragePath, err)
+			// go on
+		}
+	}
+
 	return nil
 }
 

modules/setting/actions.go

@@ -10,7 +10,8 @@ import (
 // Actions settings
 var (
 	Actions = struct {
-		Storage           // how the created logs should be stored
+		Storage                   // how the created logs should be stored
+		Artifacts         Storage // how the created artifacts should be stored
 		Enabled           bool
 		DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
 	}{
@@ -25,5 +26,9 @@ func loadActionsFrom(rootCfg ConfigProvider) {
 		log.Fatal("Failed to map Actions settings: %v", err)
 	}
 
+	actionsSec := rootCfg.Section("actions.artifacts")
+	storageType := actionsSec.Key("STORAGE_TYPE").MustString("")
+
 	Actions.Storage = getStorage(rootCfg, "actions_log", "", nil)
+	Actions.Artifacts = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec)
 }

modules/storage/storage.go

@@ -128,6 +128,8 @@ var (
 
 	// Actions represents actions storage
 	Actions ObjectStorage = uninitializedStorage
+	// Actions Artifacts represents actions artifacts storage
+	ActionsArtifacts ObjectStorage = uninitializedStorage
 )
 
 // Init init the stoarge
@@ -212,9 +214,14 @@ func initPackages() (err error) {
 func initActions() (err error) {
 	if !setting.Actions.Enabled {
 		Actions = discardStorage("Actions isn't enabled")
+		ActionsArtifacts = discardStorage("ActionsArtifacts isn't enabled")
 		return nil
 	}
 	log.Info("Initialising Actions storage with type: %s", setting.Actions.Storage.Type)
-	Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage)
+	if Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage); err != nil {
+		return err
+	}
+	log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.Artifacts.Type)
+	ActionsArtifacts, err = NewStorage(setting.Actions.Artifacts.Type, &setting.Actions.Artifacts)
 	return err
 }

options/locale/locale_en-US.ini

@@ -113,6 +113,8 @@ unknown = Unknown
 
 rss_feed = RSS Feed
 
+artifacts = Artifacts
+
 [aria]
 navbar = Navigation Bar
 footer = Footer