Skip to content

Commit

Permalink
Add retries to apple push notifications (#128)
Browse files Browse the repository at this point in the history
* Add retries to apple push notifications

* Add retry timeout config

* Add check for retry timeout being greater than send timeout
larkox authored Sep 24, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 1b3d152 commit 681c1cb
Showing 4 changed files with 70 additions and 10 deletions.
1 change: 1 addition & 0 deletions config/mattermost-push-proxy.sample.json
Original file line number Diff line number Diff line change
@@ -5,6 +5,7 @@
"ThrottleVaryByHeader":"X-Forwarded-For",
"EnableMetrics": false,
"SendTimeoutSec": 30,
"RetryTimeoutSec": 8,
"ApplePushSettings":[
{
"Type":"apple",
61 changes: 52 additions & 9 deletions server/apple_notification_server.go
Original file line number Diff line number Diff line change
@@ -25,14 +25,16 @@ type AppleNotificationServer struct {
logger *Logger
ApplePushSettings ApplePushSettings
sendTimeout time.Duration
retryTimeout time.Duration
}

func NewAppleNotificationServer(settings ApplePushSettings, logger *Logger, metrics *metrics, sendTimeoutSecs int) *AppleNotificationServer {
func NewAppleNotificationServer(settings ApplePushSettings, logger *Logger, metrics *metrics, sendTimeoutSecs int, retryTimeoutSecs int) *AppleNotificationServer {
return &AppleNotificationServer{
ApplePushSettings: settings,
metrics: metrics,
logger: logger,
sendTimeout: time.Duration(sendTimeoutSecs) * time.Second,
retryTimeout: time.Duration(retryTimeoutSecs) * time.Second,
}
}

@@ -227,15 +229,8 @@ func (me *AppleNotificationServer) SendNotification(msg *PushNotification) PushR

if me.AppleClient != nil {
me.logger.Infof("Sending apple push notification for device=%v type=%v ackId=%v", me.ApplePushSettings.Type, msg.Type, msg.AckID)
start := time.Now()

ctx, cancel := context.WithTimeout(context.Background(), me.sendTimeout)
defer cancel()

res, err := me.AppleClient.PushWithContext(ctx, notification)
if me.metrics != nil {
me.metrics.observerNotificationResponse(PushNotifyApple, time.Since(start).Seconds())
}
res, err := me.SendNotificationWithRetry(notification)
if err != nil {
me.logger.Errorf("Failed to send apple push sid=%v did=%v err=%v type=%v", msg.ServerID, msg.DeviceID, err, me.ApplePushSettings.Type)
if me.metrics != nil {
@@ -269,3 +264,51 @@ func (me *AppleNotificationServer) SendNotification(msg *PushNotification) PushR
}
return NewOkPushResponse()
}

func (me *AppleNotificationServer) SendNotificationWithRetry(notification *apns.Notification) (*apns.Response, error) {
var res *apns.Response
var err error
waitTime := time.Second

// Keep a general context to make sure the whole retry
// doesn't take longer than the timeout.
generalContext, cancelGeneralContext := context.WithTimeout(context.Background(), me.sendTimeout)
defer cancelGeneralContext()

for retries := 0; retries < MAX_RETRIES; retries++ {
start := time.Now()

retryContext, cancelRetryContext := context.WithTimeout(generalContext, me.retryTimeout)
defer cancelRetryContext()
res, err = me.AppleClient.PushWithContext(retryContext, notification)
if me.metrics != nil {
me.metrics.observerNotificationResponse(PushNotifyApple, time.Since(start).Seconds())
}

if err == nil {
break
}

me.logger.Errorf("Failed to send apple push did=%v retry=%v error=%v", notification.DeviceToken, retries, err)

if retries == MAX_RETRIES-1 {
me.logger.Errorf("Max retries reached did=%v", notification.DeviceToken)
break
}

select {
case <-generalContext.Done():
case <-time.After(waitTime):
}

if generalContext.Err() != nil {
me.logger.Infof("Not retrying because context error did=%v retry=%v error=%v", notification.DeviceToken, retries, generalContext.Err())
err = generalContext.Err()
break
}

waitTime *= 2
}

return res, err
}
15 changes: 15 additions & 0 deletions server/config_push_proxy.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ type ConfigPushProxy struct {
ThrottleVaryByHeader string
LogFileLocation string
SendTimeoutSec int
RetryTimeoutSec int
ApplePushSettings []ApplePushSettings
EnableMetrics bool
EnableConsoleLog bool
@@ -81,6 +82,20 @@ func LoadConfig(fileName string) (*ConfigPushProxy, error) {
if !cfg.EnableConsoleLog && !cfg.EnableFileLog {
cfg.EnableConsoleLog = true
}

// Set timeout defaults
if cfg.SendTimeoutSec == 0 {
cfg.SendTimeoutSec = 30
}

if cfg.RetryTimeoutSec == 0 {
cfg.RetryTimeoutSec = 8
}

if cfg.RetryTimeoutSec > cfg.SendTimeoutSec {
cfg.RetryTimeoutSec = cfg.SendTimeoutSec
}

if cfg.EnableFileLog {
if cfg.LogFileLocation == "" {
// We just do an mkdir -p equivalent.
3 changes: 2 additions & 1 deletion server/server.go
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@ const (
HEADER_REAL_IP = "X-Real-IP"
WAIT_FOR_SERVER_SHUTDOWN = time.Second * 5
CONNECTION_TIMEOUT_SECONDS = 60
MAX_RETRIES = 3
)

type NotificationServer interface {
@@ -69,7 +70,7 @@ func (s *Server) Start() {
}

for _, settings := range s.cfg.ApplePushSettings {
server := NewAppleNotificationServer(settings, s.logger, m, s.cfg.SendTimeoutSec)
server := NewAppleNotificationServer(settings, s.logger, m, s.cfg.SendTimeoutSec, s.cfg.RetryTimeoutSec)
err := server.Initialize()
if err != nil {
s.logger.Errorf("Failed to initialize client: %v", err)

0 comments on commit 681c1cb

Please sign in to comment.