Skip to content

Commit

Permalink
feat: cross account caching with role (#336)
Browse files Browse the repository at this point in the history
This allows to have a single account for caching in multi-account
environments.
Changes:

- Added support to assume role
- Added the ability to specify access policy
- Added the ability to specify lifecycle policy


Initially, I had two approaches to make this work:
1) let other accounts create repos and fill them with images
2) use the role in the target account (this PR)

While 1) would be preferable, unfortunately, it's doesn't look like it's
possible: you can allow other accounts to create repos, but not put any
policies.
  • Loading branch information
BEvgeniyS authored Sep 7, 2022
1 parent 730fcaa commit 98d138e
Show file tree
Hide file tree
Showing 7 changed files with 228 additions and 31 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,5 @@
# vendor/

.idea/
coverage.txt
k8s-image-swapper
44 changes: 44 additions & 0 deletions .k8s-image-swapper.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ target:
aws:
accountId: 123456789
region: ap-southeast-2
role: arn:aws:iam::123456789012:role/roleName
ecrOptions:
tags:
- key: CreatedBy
Expand All @@ -51,5 +52,48 @@ target:
encryptionConfiguration:
encryptionType: AES256
kmsKey: string
accessPolicy: |
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "AllowCrossAccountPull",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
],
"Condition": {
"StringEquals": {
"aws:PrincipalOrgID": [
"o-xxxxxxxx"
]
}
}
}
]
}
lifecyclePolicy: |
{
"rules": [
{
"rulePriority": 1,
"description": "Rule 1",
"selection": {
"tagStatus": "any",
"countType": "imageCountMoreThan",
"countNumber": 1
},
"action": {
"type": "expire"
}
}
]
}
# dockerio:
# quayio:
2 changes: 1 addition & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
//metricsRec := metrics.NewPrometheus(promReg)
log.Trace().Interface("config", cfg).Msg("config")

rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain())
rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain(), cfg.Target.AWS.AccountID, cfg.Target.AWS.Role, cfg.Target.AWS.AccessPolicy, cfg.Target.AWS.LifecyclePolicy)
if err != nil {
log.Err(err).Msg("error connecting to registry client")
os.Exit(1)
Expand Down
85 changes: 85 additions & 0 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,91 @@ Choose from one of the strategies below or an alternative if needed.
--from-literal=aws_secret_access_key=<...>
```

#### Using ECR registries cross-account

Although ECR allows creating registry policy that allows reposistories creation from different account, there's no way to push anything to these repositories.
ECR resource-level policy can not be applied during creation, and to apply it afterwards we need ecr:SetRepositoryPolicy permission, which foreign account doesn't have.

One way out of this conundrum is to assume the role in target account

```yaml
target:
type: aws
aws:
accountId: 123456789
region: ap-southeast-2
role: arn:aws:iam::123456789012:role/roleName
```
!!! note
Make sure that target role has proper trust permissions that allow to assume it cross-account

!!! note
In order te be able to pull images from outside accounts, you will have to apply proper access policy


#### Access policy

You can specify the access policy that will be applied to the created repos in config. Policy should be raw json string.
For example:
```yaml
target:
aws:
accountId: 123456789
region: ap-southeast-2
role: arn:aws:iam::123456789012:role/roleName
accessPolicy: '{
"Statement": [
{
"Sid": "AllowCrossAccountPull",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": [
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"ecr:BatchCheckLayerAvailability"
],
"Condition": {
"StringEquals": {
"aws:PrincipalOrgID": "o-xxxxxxxxxx"
}
}
}
],
"Version": "2008-10-17"
}'
```

#### Lifecycle policy

Similarly to access policy, lifecycle policy can be specified, for example:

```yaml
target:
aws:
accountId: 123456789
region: ap-southeast-2
role: arn:aws:iam::123456789012:role/roleName
accessPolicy: '{
"rules": [
{
"rulePriority": 1,
"description": "Rule 1",
"selection": {
"tagStatus": "any",
"countType": "imageCountMoreThan",
"countNumber": 1000
},
"action": {
"type": "expire"
}
}
]
}
'
```

#### Service Account

1. Create an Webidentity IAM role (e.g. `k8s-image-swapper`) with the following trust policy, e.g
Expand Down
7 changes: 5 additions & 2 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,11 @@ type Target struct {
}

type AWS struct {
AccountID string `yaml:"accountId"`
Region string `yaml:"region"`
AccountID string `yaml:"accountId"`
Region string `yaml:"region"`
Role string `yaml:"role"`
AccessPolicy string `yaml:"accessPolicy"`
LifecyclePolicy string `yaml:"lifecyclePolicy"`
}

func (a *AWS) EcrDomain() string {
Expand Down
111 changes: 85 additions & 26 deletions pkg/registry/ecr.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/aws/aws-sdk-go/service/ecr/ecriface"
Expand All @@ -19,11 +20,14 @@ import (
var execCommand = exec.Command

type ECRClient struct {
client ecriface.ECRAPI
ecrDomain string
authToken []byte
cache *ristretto.Cache
scheduler *gocron.Scheduler
client ecriface.ECRAPI
ecrDomain string
authToken []byte
cache *ristretto.Cache
scheduler *gocron.Scheduler
targetAccount string
accessPolicy string
lifecyclePolicy string
}

func (e *ECRClient) Credentials() string {
Expand All @@ -41,13 +45,15 @@ func (e *ECRClient) CreateRepository(name string) error {
ScanOnPush: aws.Bool(true),
},
ImageTagMutability: aws.String(ecr.ImageTagMutabilityMutable),
RegistryId: &e.targetAccount,
Tags: []*ecr.Tag{
{
Key: aws.String("CreatedBy"),
Value: aws.String("k8s-image-swapper"),
},
},
})

if err != nil {
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {
Expand All @@ -63,6 +69,37 @@ func (e *ECRClient) CreateRepository(name string) error {
}
}

if len(e.accessPolicy) > 0 {
log.Info().Msg("Setting access policy on" + name)
log.Debug().Msg("Access policy: \n" + e.accessPolicy)
_, err := e.client.SetRepositoryPolicy(&ecr.SetRepositoryPolicyInput{
PolicyText: &e.accessPolicy,
RegistryId: &e.targetAccount,
RepositoryName: aws.String(name),
})

if err != nil {
log.Err(err).Msg(err.Error())
return err
}
}

if len(e.lifecyclePolicy) > 0 {
log.Info().Msg("Setting lifecycle policy on" + name)
log.Debug().Msg("Lifecycle policy: \n" + e.lifecyclePolicy)

_, err := e.client.PutLifecyclePolicy(&ecr.PutLifecyclePolicyInput{
LifecyclePolicyText: &e.lifecyclePolicy,
RegistryId: &e.targetAccount,
RepositoryName: aws.String(name),
})

if err != nil {
log.Err(err).Msg(err.Error())
return err
}
}

e.cache.Set(name, "", 1)

return nil
Expand Down Expand Up @@ -115,7 +152,10 @@ func (e *ECRClient) Endpoint() string {

// requestAuthToken requests and returns an authentication token from ECR with its expiration date
func (e *ECRClient) requestAuthToken() ([]byte, time.Time, error) {
getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{})
getAuthTokenOutput, err := e.client.GetAuthorizationToken(&ecr.GetAuthorizationTokenInput{
RegistryIds: []*string{&e.targetAccount},
})

if err != nil {
return []byte(""), time.Time{}, err
}
Expand Down Expand Up @@ -146,18 +186,33 @@ func (e *ECRClient) scheduleTokenRenewal() error {
return nil
}

func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
sess := session.Must(session.NewSessionWithOptions(session.Options{
func NewECRClient(region string, ecrDomain string, targetAccount string, role string, accessPolicy string, lifecyclePolicy string) (*ECRClient, error) {
var sess *session.Session
var config *aws.Config
if role != "" {
log.Debug().Msg("Role is specified. Assuming " + role)
stsSession, _ := session.NewSession(config)
creds := stscreds.NewCredentials(stsSession, role)
config = aws.NewConfig().
WithRegion(region).
WithCredentialsChainVerboseErrors(true).
WithHTTPClient(&http.Client{
Timeout: 3 * time.Second,
}).
WithCredentials(creds)
} else {
config = aws.NewConfig().
WithRegion(region).
WithCredentialsChainVerboseErrors(true).
WithHTTPClient(&http.Client{
Timeout: 3 * time.Second,
})
}

sess = session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: (*config),
}))

config := aws.NewConfig().
WithRegion(region).
WithCredentialsChainVerboseErrors(true).
WithHTTPClient(&http.Client{
Timeout: 3 * time.Second,
})

ecrClient := ecr.New(sess, config)

cache, err := ristretto.NewCache(&ristretto.Config{
Expand All @@ -173,10 +228,13 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
scheduler.StartAsync()

client := &ECRClient{
client: ecrClient,
ecrDomain: ecrDomain,
cache: cache,
scheduler: scheduler,
client: ecrClient,
ecrDomain: ecrDomain,
cache: cache,
scheduler: scheduler,
targetAccount: targetAccount,
accessPolicy: accessPolicy,
lifecyclePolicy: lifecyclePolicy,
}

if err := client.scheduleTokenRenewal(); err != nil {
Expand All @@ -186,13 +244,14 @@ func NewECRClient(region string, ecrDomain string) (*ECRClient, error) {
return client, nil
}

func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string) (*ECRClient, error) {
func NewMockECRClient(ecrClient ecriface.ECRAPI, region string, ecrDomain string, targetAccount, role string) (*ECRClient, error) {
client := &ECRClient{
client: ecrClient,
ecrDomain: ecrDomain,
cache: nil,
scheduler: nil,
authToken: []byte("mock-ecr-client-fake-auth-token"),
client: ecrClient,
ecrDomain: ecrDomain,
cache: nil,
scheduler: nil,
targetAccount: targetAccount,
authToken: []byte("mock-ecr-client-fake-auth-token"),
}

return client, nil
Expand Down
Loading

0 comments on commit 98d138e

Please sign in to comment.