Skip to content

Commit

Permalink
ebs br: new snapshot tagging (#50548) (#50569)
Browse files Browse the repository at this point in the history
close #50559
  • Loading branch information
ti-chi-bot authored Jan 18, 2024
1 parent e850d08 commit c157388
Showing 1 changed file with 13 additions and 28 deletions.
41 changes: 13 additions & 28 deletions br/pkg/aws/ebs.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package aws
import (
"context"
"fmt"
"math"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -74,6 +73,10 @@ func (e *EC2Session) CreateSnapshots(backupInfo *config.EBSBasedBRMeta) (map[str
}
}

tags := []*ec2.Tag{
ec2Tag("TiDBCluster-BR", "new"),
}

workerPool := utils.NewWorkerPool(e.concurrency, "create snapshots")
for i := range backupInfo.TiKVComponent.Stores {
store := backupInfo.TiKVComponent.Stores[i]
Expand Down Expand Up @@ -137,6 +140,12 @@ func (e *EC2Session) CreateSnapshots(backupInfo *config.EBSBasedBRMeta) (map[str
createSnapshotInput.SetInstanceSpecification(&instanceSpecification)
// Copy tags from source volume
createSnapshotInput.SetCopyTagsFromSource("volume")
createSnapshotInput.SetTagSpecifications([]*ec2.TagSpecification{
{
ResourceType: aws.String(ec2.ResourceTypeSnapshot),
Tags: tags,
},
})
resp, err := e.createSnapshotsWithRetry(context.TODO(), &createSnapshotInput)

if err != nil {
Expand Down Expand Up @@ -330,11 +339,6 @@ func (e *EC2Session) EnableDataFSR(meta *config.EBSBasedBRMeta, targetAZ string)

// waitDataFSREnabled waits FSR for data volume snapshots are all enabled and also have enough credit balance
func (e *EC2Session) waitDataFSREnabled(snapShotIDs []*string, targetAZ string) error {
// Record current time
start := time.Now()

// get the maximum size of volumes, in GiB
var maxVolumeSize int64 = 0
resp, err := e.ec2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{SnapshotIds: snapShotIDs})
if err != nil {
return errors.Trace(err)
Expand All @@ -343,25 +347,7 @@ func (e *EC2Session) waitDataFSREnabled(snapShotIDs []*string, targetAZ string)
return errors.Errorf("specified snapshot [%s] is not found", *snapShotIDs[0])
}

for _, s := range resp.Snapshots {
if *s.VolumeSize > maxVolumeSize {
maxVolumeSize = *s.VolumeSize
}
}

// Calculate the time in minutes to fill 1.0 credit according to
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-fast-snapshot-restore.html#volume-creation-credits
// 5 minutes more is just for safe
fillElapsedTime := 60.0/(math.Min(10, 1024.0/(float64)(maxVolumeSize))) + 5

// We have to sleep for at least fillElapsedTime minutes in order to make credits are filled to 1.0
// Let's heartbeat every 5 minutes
for time.Since(start) <= time.Duration(fillElapsedTime)*time.Minute {
log.Info("FSR enablement is ongoing, going to sleep for 5 minutes...")
time.Sleep(5 * time.Minute)
}

// Wait that all snapshot has enough fsr credit balance, it's very likely true since we have wait for long enough
// Wait that all snapshot has enough fsr credit balance
log.Info("Start check and wait all snapshots have enough fsr credit balance")

startIdx := 0
Expand All @@ -379,9 +365,8 @@ func (e *EC2Session) waitDataFSREnabled(snapShotIDs []*string, targetAZ string)
}
retryCount++
}
// Retry for both invalid calling and not enough fsr credit
// Cloudwatch by default flushes every 5 seconds. So, 20 seconds wait should be enough
time.Sleep(20 * time.Second)
// Retry for both invalid calling and not enough fsr credit at 3 minute intervals
time.Sleep(3 * time.Minute)
}
}

Expand Down

0 comments on commit c157388

Please sign in to comment.