-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add temporal prototype and readme. (#190)
* Add temporal prototype and readme. * Delete unnecessary file. * Add support for sessions and configurable data dirs.
- Loading branch information
1 parent
7e2e205
commit a6b6443
Showing
10 changed files
with
814 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| # Deployment Prototype via Temporal | ||
|
|
||
| This workflow emulates a typical deployment system where there exists a deployment queue for a single repository and each revision that needs to be deployed in order. The workflow iterates through that queue serially and runs the dry run steps (terraform init, plan). Approval is required to run the terraform apply operation. | ||
|
|
||
| terraform data (plan files/archives) are kept locally and worker sessions are used to ensure that terraform activities within a given workflow are run on the same worker and therefore access the same data directory. | ||
|
|
||
| ## Setup | ||
|
|
||
| In order to run this workflow a temporal cluster needs to be running. I usually just run a local version of this: | ||
|
|
||
| ``` | ||
| git clone git@github.com/danielhochman/docker-compose | ||
| cd docker-compose | ||
| docker-compose up -d | ||
| ``` | ||
|
|
||
| Next we'll want to start the worker: | ||
|
|
||
| ``` | ||
| go run main.go worker --ghuser nishkrishnan --ghtoken <GITHUB_ACCESS_TOKEN> | ||
| ``` | ||
|
|
||
| Finally we'll want to start the application server which is responsible for translating api requests to workflow executions/signals: | ||
|
|
||
| ``` | ||
| go run main.go application-server | ||
| ``` | ||
|
|
||
| Now we are ready to start making requests to the server. | ||
|
|
||
| ## Request Types | ||
|
|
||
| Deploy/Queue a revision | ||
|
|
||
| ``` | ||
| curl -H 'Content-Type: application/json' -d '{ | ||
| "Repo": { | ||
| "Owner": "<OWNER>", | ||
| "Name" : "<REPO>" | ||
| }, | ||
| "Branch": "<BRANCH>", | ||
| "Revision" : "<SHA>" | ||
| }' localhost:9000/api/deploy | ||
| ``` | ||
|
|
||
| Approve a deployment | ||
|
|
||
| ``` | ||
| curl -H 'Content-Type: application/json' -d '{ | ||
| "User": "<USER>", | ||
| "Status": 0, | ||
| "RunID": "<RUN_ID>", | ||
| "WorkflowID" : "<SHA>" | ||
| }' localhost:9000/api/plan_review | ||
| ``` | ||
|
|
||
| Note: run id can be found by looking at the worker logs, in an ideal world this info is relayed back to the client. | ||
|
|
||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| package activities | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/runatlantis/atlantis/server/events/models" | ||
| "go.temporal.io/sdk/activity" | ||
| ) | ||
|
|
||
| type CloneActivityRequest struct { | ||
| Repo models.Repo | ||
| Branch string | ||
| Dir string | ||
| Revision string | ||
| } | ||
|
|
||
| type CloneActivityResponse struct { | ||
| Dir string | ||
| } | ||
|
|
||
| func Clone(ctx context.Context, request CloneActivityRequest) (CloneActivityResponse, error) { | ||
| log := activity.GetLogger(ctx) | ||
|
|
||
| cloneDir := request.Dir | ||
| headRepo := request.Repo | ||
|
|
||
| err := os.RemoveAll(cloneDir) | ||
| if err != nil { | ||
| return CloneActivityResponse{}, errors.Wrapf(err, "deleting dir %q before cloning", cloneDir) | ||
| } | ||
|
|
||
| // Create the directory and parents if necessary. | ||
| log.Info("creating dir %q", cloneDir) | ||
| if err := os.MkdirAll(cloneDir, 0700); err != nil { | ||
| return CloneActivityResponse{}, errors.Wrap(err, "creating new workspace") | ||
| } | ||
|
|
||
| headCloneURL := headRepo.CloneURL | ||
|
|
||
| var cmds = [][]string{ | ||
| { | ||
| "git", "clone", "--branch", request.Branch, "--single-branch", headCloneURL, cloneDir, | ||
| }, | ||
| { | ||
| "git", "checkout", request.Revision, | ||
| }, | ||
| } | ||
|
|
||
| for _, args := range cmds { | ||
| cmd := exec.Command(args[0], args[1:]...) // nolint: gosec | ||
| cmd.Dir = cloneDir | ||
| // The git merge command requires these env vars are set. | ||
| cmd.Env = append(os.Environ(), []string{ | ||
| "EMAIL=atlantis@runatlantis.io", | ||
| "GIT_AUTHOR_NAME=atlantis", | ||
| "GIT_COMMITTER_NAME=atlantis", | ||
| }...) | ||
|
|
||
| cmdStr := sanitizeGitCredentials(strings.Join(cmd.Args, " "), headRepo) | ||
| output, err := cmd.CombinedOutput() | ||
| sanitizedOutput := sanitizeGitCredentials(string(output), headRepo) | ||
| if err != nil { | ||
| sanitizedErrMsg := sanitizeGitCredentials(err.Error(), headRepo) | ||
| return CloneActivityResponse{}, fmt.Errorf("running %s: %s: %s", cmdStr, sanitizedOutput, sanitizedErrMsg) | ||
| } | ||
| log.Debug("ran: %s. Output: %s", cmdStr, strings.TrimSuffix(sanitizedOutput, "\n")) | ||
| } | ||
| return CloneActivityResponse{Dir: cloneDir}, nil | ||
| } | ||
|
|
||
| // sanitizeGitCredentials replaces any git clone urls that contain credentials | ||
| // in s with the sanitized versions. | ||
| func sanitizeGitCredentials(s string, head models.Repo) string { | ||
| return strings.Replace(s, head.CloneURL, head.SanitizedCloneURL, -1) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package activities | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| type InitRequest struct { | ||
| RootDir string | ||
| } | ||
|
|
||
| type InitResponse struct { | ||
| Output string | ||
| } | ||
|
|
||
| type PlanRequest struct { | ||
| RootDir string | ||
| } | ||
|
|
||
| type PlanResponse struct { | ||
| Output string | ||
| Planfile string | ||
| } | ||
|
|
||
| type ApplyRequest struct { | ||
| RootDir string | ||
| Planfile string | ||
| } | ||
|
|
||
| type ApplyResponse struct { | ||
| Output string | ||
| } | ||
|
|
||
| func Init(ctx context.Context, request InitRequest) (InitResponse, error) { | ||
| output, err := terraform(request.RootDir, "init", "-input=false") | ||
|
|
||
| // output using fmt instead of logger to get pretty formatting. | ||
| // this is probably susceptible to duplication though in the event of replays | ||
| // this is only for prototype purposes anyways. | ||
| fmt.Println(string(output)) | ||
|
|
||
| if err != nil { | ||
| return InitResponse{}, errors.Wrap(err, "running terraform init") | ||
| } | ||
|
|
||
| return InitResponse{Output: string(output)}, nil | ||
| } | ||
|
|
||
| func Plan(ctx context.Context, request PlanRequest) (PlanResponse, error) { | ||
| planFile := "plan.tfplan" | ||
|
|
||
| output, err := terraform(request.RootDir, "plan", "-input=false", "-refresh", "-out", fmt.Sprintf("%q", planFile)) | ||
|
|
||
| // output using fmt instead of logger to get pretty formatting. | ||
| // this is probably susceptible to duplication though in the event of replays | ||
| // this is only for prototype purposes anyways. | ||
| fmt.Println(string(output)) | ||
|
|
||
| if err != nil { | ||
| return PlanResponse{}, errors.Wrap(err, "running terraform plan") | ||
| } | ||
|
|
||
| return PlanResponse{ | ||
| Output: string(output), | ||
| Planfile: planFile, | ||
| }, nil | ||
| } | ||
|
|
||
| func Apply(ctx context.Context, request ApplyRequest) (ApplyResponse, error) { | ||
| output, err := terraform(request.RootDir, "apply", "-input=false", filepath.Join(request.RootDir, request.Planfile)) | ||
|
|
||
| // output using fmt instead of logger to get pretty formatting. | ||
| // this is probably susceptible to duplication though in the event of replays | ||
| // this is only for prototype purposes anyways. | ||
| fmt.Println(string(output)) | ||
|
|
||
| if err != nil { | ||
| return ApplyResponse{}, errors.Wrap(err, "running terraform apply") | ||
| } | ||
|
|
||
| return ApplyResponse{Output: string(output)}, nil | ||
| } | ||
|
|
||
| func terraform(dir string, args ...string) ([]byte, error) { | ||
| tfCmd := fmt.Sprintf("terraform %s", strings.Join(args, " ")) | ||
| cmd := exec.Command("sh", "-c", tfCmd) | ||
| cmd.Dir = dir | ||
| cmd.Env = os.Environ() | ||
|
|
||
| return cmd.CombinedOutput() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package activities | ||
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
|
|
||
| "github.com/google/go-github/v31/github" | ||
| "github.com/pkg/errors" | ||
| "github.com/runatlantis/atlantis/server/events" | ||
| "github.com/runatlantis/atlantis/server/events/models" | ||
| ) | ||
|
|
||
| type VCSClientWrapper struct { | ||
| client *github.Client | ||
| eventParser *events.EventParser | ||
| } | ||
|
|
||
| func NewVCSClientWrapper(githubUser string, githubToken string) *VCSClientWrapper { | ||
| eventParser := &events.EventParser{ | ||
| GithubUser: githubUser, | ||
| GithubToken: githubToken, | ||
| } | ||
|
|
||
| transport := &github.BasicAuthTransport{ | ||
| Username: strings.TrimSpace(githubUser), | ||
| Password: strings.TrimSpace(githubToken), | ||
| } | ||
|
|
||
| client := github.NewClient(transport.Client()) | ||
|
|
||
| return &VCSClientWrapper{ | ||
| client: client, | ||
| eventParser: eventParser, | ||
| } | ||
| } | ||
|
|
||
| type GetRepositoryRequest struct { | ||
| Owner string | ||
| Repo string | ||
| } | ||
|
|
||
| type GetRepositoryResponse struct { | ||
| Repo models.Repo | ||
| } | ||
|
|
||
| func (r *VCSClientWrapper) GetRepository(ctx context.Context, request GetRepositoryRequest) (GetRepositoryResponse, error) { | ||
| rawRepo, _, err := r.client.Repositories.Get(ctx, request.Owner, request.Repo) | ||
|
|
||
| if err != nil { | ||
| return GetRepositoryResponse{}, errors.Wrapf(err, "getting github repo %s/%s", request.Owner, request.Repo) | ||
| } | ||
|
|
||
| repository, err := r.eventParser.ParseGithubRepo(rawRepo) | ||
|
|
||
| if err != nil { | ||
| return GetRepositoryResponse{}, errors.Wrapf(err, "parsing github repo %s from response", *rawRepo.Name) | ||
| } | ||
|
|
||
| return GetRepositoryResponse{Repo: repository}, err | ||
| } |
Oops, something went wrong.