Merge pull request #267 from codekitchen/failure-backoff

Failure backoff

Close #30

lib/raven/client.rb

@@ -19,6 +19,7 @@ class Client
     def initialize(configuration)
       @configuration = configuration
       @processors = configuration.processors.map { |v| v.new(self) }
+      @state = ClientState.new
     end
 
     def send(event)
@@ -29,18 +30,25 @@ def send(event)
 
       # Set the project ID correctly
       event.project = self.configuration.project_id
+
+      if !@state.should_try?
+        Raven.logger.error("Not sending event due to previous failure(s): #{get_log_message(event)}")
+        return
+      end
+
       Raven.logger.debug "Sending event #{event.id} to Sentry"
 
       content_type, encoded_data = encode(event)
       begin
         transport.send(generate_auth_header, encoded_data,
                        :content_type => content_type)
       rescue => e
-        Raven.logger.error "Unable to record event with remote Sentry server (#{e.class} - #{e.message})"
-        e.backtrace[0..10].each { |line| Raven.logger.error(line) }
+        failed_send(e, event)
         return
       end
 
+      successful_send
+
       event
     end
 
@@ -66,6 +74,10 @@ def encode(event)
       end
     end
 
+    def get_log_message(event)
+      (event && event.message) || '<no message value>'
+    end
+
     def transport
       @transport ||=
         case self.configuration.scheme
@@ -100,5 +112,53 @@ def strict_encode64(string)
       end
     end
 
+    def successful_send
+      @state.success
+    end
+
+    def failed_send(e, event)
+      @state.failure
+      Raven.logger.error "Unable to record event with remote Sentry server (#{e.class} - #{e.message})"
+      e.backtrace[0..10].each { |line| Raven.logger.error(line) }
+      Raven.logger.error("Failed to submit event: #{get_log_message(event)}")
+    end
+
+  end
+
+  class ClientState
+    def initialize
+      reset
+    end
+
+    def should_try?
+      return true if @status == :online
+
+      interval = @retry_after || [@retry_number, 6].min ** 2
+      return true if Time.now - @last_check >= interval
+
+      false
+    end
+
+    def failure(retry_after = nil)
+      @status = :error
+      @retry_number += 1
+      @last_check = Time.now
+      @retry_after = retry_after
+    end
+
+    def success
+      reset
+    end
+
+    def reset
+      @status = :online
+      @retry_number = 0
+      @last_check = nil
+      @retry_after = nil
+    end
+
+    def failed?
+      @status == :error
+    end
   end
 end

sentry-raven.gemspec

@@ -23,4 +23,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "coveralls"
   gem.add_development_dependency "rest-client", "< 1.7.0" if RUBY_VERSION == '1.8.7'
   gem.add_development_dependency "rest-client" if RUBY_VERSION > '1.8.7'
+  gem.add_development_dependency "timecop", "0.6.1" if RUBY_VERSION == '1.8.7'
+  gem.add_development_dependency "timecop" if RUBY_VERSION > '1.8.7'
 end

spec/raven/client_state_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require 'timecop'
+
+describe Raven::ClientState do
+  let(:state) { Raven::ClientState.new }
+
+  it 'should try when online' do
+    expect(state.should_try?).to eq(true)
+  end
+
+  it 'should not try with a new error' do
+    state.failure
+    expect(state.should_try?).to eq(false)
+  end
+
+  it 'should try again after time passes' do
+    Timecop.freeze(-10) { state.failure }
+    expect(state.should_try?).to eq(true)
+  end
+
+  it 'should try again after success' do
+    state.failure
+    state.success
+    expect(state.should_try?).to eq(true)
+  end
+
+  it 'should try again after retry_after' do
+    Timecop.freeze(-2) { state.failure(1) }
+    expect(state.should_try?).to eq(true)
+  end
+
+  it 'should exponentially backoff' do
+    Timecop.freeze do
+      state.failure
+      Timecop.travel(2)
+      expect(state.should_try?).to eq(true)
+
+      state.failure
+      Timecop.travel(3)
+      expect(state.should_try?).to eq(false)
+      Timecop.travel(2)
+      expect(state.should_try?).to eq(true)
+
+      state.failure
+      Timecop.travel(8)
+      expect(state.should_try?).to eq(false)
+      Timecop.travel(2)
+      expect(state.should_try?).to eq(true)
+    end
+  end
+end

spec/raven/integration_spec.rb

@@ -58,4 +58,19 @@
     stubs.verify_stubbed_calls
 
   end
+
+  example "timed backoff should prevent sends" do
+    Raven.configure do |config|
+      config.server = 'http://12345:67890@sentry.localdomain/sentry/42'
+      config.environments = ["test"]
+      config.current_environment = "test"
+      config.http_adapter = [:test, nil]
+    end
+
+    expect_any_instance_of(Raven::Transports::HTTP).to receive(:send).exactly(1).times.and_raise(Faraday::Error::ConnectionFailed, "conn failed")
+    expect { Raven.capture_exception(build_exception) }.not_to raise_error
+
+    expect(Raven.logger).to receive(:error).exactly(1).times
+    expect { Raven.capture_exception(build_exception) }.not_to raise_error
+  end
 end

spec/raven/integrations/rack_spec.rb

@@ -60,7 +60,7 @@
   it 'should allow empty rack env in rspec tests' do
     env = {} # the rack env is empty when running rails/rspec tests
     Raven.rack_context(env)
-    expect { Raven.capture_exception(build_exception()) }.not_to raise_error
+    expect { Raven.capture_exception(build_exception) }.not_to raise_error
   end
 
   it 'should bind request context' do