Support for non-deterministic resource configuration (#485)

* Support non-deterministic resource configuration

This behavior allows the HTTP adapter to call the configuration block on
every request.

Used in combination with external variables, such as thread local
variable, it enables the configuration of a single host/port combination
as multiple resources.

```ruby
SEMIAN_PARAMETERS = {
  # ...
  dynamic: true,
}

class CurrentSemianSubResource < ActiveSupport::Attributes
 attribute :name
end

Semian::NetHTTP.semian_configuration = proc do |host, port|
  name = "#{host}_#{port}"
  if (sub_resource_name = CurrentSemianSubResource.name)
    name << "_#{name}"
  end
  SEMIAN_PARAMETERS.merge(name: name)
end

# Two requests to example.com can use two different semian resources,
# as long as `CurrentSemianSubResource.name` is set accordingly:
# CurrentSemianSubResource.set(name: "sub_resource_1") { Net::HTTP.get_response(URI("http://example.com")) }
# and:
# CurrentSemianSubResource.set(name: "sub_resource_2") { Net::HTTP.get_response(URI("http://example.com")) }
```

* Add Net::HTTP adapter with dynamic config example

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Unreleased
 
 * Redis Readonly errors won't trigger semian open circuit. (#489)
+* New `dynamic` options, allowing a per-request resource configuration with the HTTP adapter. (#485)
 
 # v0.18.1
 

README.md

@@ -254,10 +254,40 @@ The `semian_options` passed apply to that resource. Semian creates the `semian_i
 from the `name` to look up and store changes in the circuit breaker and bulkhead states
 and associate successes, failures, errors with the protected resource.
 
-We only require that:
-* the `semian_configuration` be **set only once** over the lifetime of the library
-* the output of the `proc` be the same over time, that is, the configuration produced by
-  each pair of `host`, `port` is **the same each time** the callback is invoked.
+We only require that the `semian_configuration` be **set only once** over the lifetime of
+the library.
+
+If you need to return different values for the same pair of `host`/`port` value, you **must**
+include the `dynamic: true` option. Returning different values for the same `host`/`port` values
+without setting the `dynamic` option can lead to undesirable behavior.
+
+A common example for dynamic options is the use of a thread local variable, such as
+`ActiveSupport::CurrentAttributes`, for requests to a service acting as a proxy.
+
+```ruby
+SEMIAN_PARAMETERS = {
+  # ...
+  dynamic: true,
+}
+
+class CurrentSemianSubResource < ActiveSupport::Attributes
+ attribute :name
+end
+
+Semian::NetHTTP.semian_configuration = proc do |host, port|
+  name = "#{host}_#{port}"
+  if (sub_resource_name = CurrentSemianSubResource.name)
+    name << "_#{name}"
+  end
+  SEMIAN_PARAMETERS.merge(name: name)
+end
+
+# Two requests to example.com can use two different semian resources,
+# as long as `CurrentSemianSubResource.name` is set accordingly:
+# CurrentSemianSubResource.set(name: "sub_resource_1") { Net::HTTP.get_response(URI("http://example.com")) }
+# and:
+# CurrentSemianSubResource.set(name: "sub_resource_2") { Net::HTTP.get_response(URI("http://example.com")) }
+```
 
 For most purposes, `"#{host}_#{port}"` is a good default `name`. Custom `name` formats
 can be useful to grouping related subdomains as one resource, so that they all

examples/net_http/07_circuit_dynamic_config.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "semian"
+require "semian/net_http"
+require_relative "../colors"
+
+puts "> Starting example #{__FILE__}".blue.bold
+puts
+puts "> Initialize print Semian state changes".blue.bold
+Semian.subscribe do |event, resource, scope, adapter|
+  puts "[semian] adapter=#{adapter} scope=#{scope} event=#{event} " \
+    "resource_name=#{resource.name} resource=#{resource}".gray
+end
+
+SEMIAN_PARAMETERS = {
+  circuit_breaker: true,
+  success_threshold: 3,
+  error_threshold: 1,
+  error_timeout: 5,
+  bulkhead: false,
+  dynamic: true,
+  open_circuit_server_errors: true,
+}
+
+uri = URI("http://example.com:80")
+
+puts "> Configure Circuit breaker for Net::HTTP".blue.bold
+Semian::NetHTTP.semian_configuration = proc do |host, port|
+  puts "[semian/http] invoked config for host=#{host}(#{host.class}) port=#{port}(#{port.class})".gray
+
+  if host == "example.com"
+    puts "  set resource name example_com".gray
+    sub_resource_name = Thread.current[:current_semian_sub_resource_name]
+    # We purposefully do not use the port as the resource name, so that we can
+    # force the circuit to open by sending a request to an invalid port, e.g. 81
+    SEMIAN_PARAMETERS.merge(name: "example_com_#{sub_resource_name}")
+  else
+    puts "  skip semian initialization".gray
+    nil
+  end
+end
+
+puts "> Test requests".blue.bold
+puts " >> 1. Request to http://example.com - success".cyan
+Thread.current[:current_semian_sub_resource_name] = "sub_resource_1"
+response = Net::HTTP.get_response(uri)
+puts "  > Response status: #{response.code}"
+puts
+
+puts " >> 2. Request to http://example.com - success".cyan
+Thread.current[:current_semian_sub_resource_name] = "sub_resource_2"
+response = Net::HTTP.get_response(uri)
+puts "  > Response status: #{response.code}"
+puts
+
+puts "> Review semian state:".blue.bold
+resource1 = Semian["nethttp_example_com_sub_resource_1"]
+puts "resource_name=#{resource1.name} resource=#{resource1} " \
+  "closed=#{resource1.closed?} open=#{resource1.open?} " \
+  "half_open=#{resource1.half_open?}".gray
+resource2 = Semian["nethttp_example_com_sub_resource_2"]
+puts "resource_name=#{resource2.name} resource=#{resource2} " \
+  "closed=#{resource2.closed?} open=#{resource2.open?} " \
+  "half_open=#{resource2.half_open?}".gray
+puts
+
+puts "> Test request errors".blue.bold
+puts " >> 3. Request to http://example.com - fail".magenta
+Thread.current[:current_semian_sub_resource_name] = "sub_resource_1"
+begin
+  # We use a different port to make the connection fail
+  Net::HTTP.start(uri.host, 81, open_timeout: 1) do |http|
+    http.request_get(uri)
+  end
+rescue => e
+  puts "   >> Could not connect: #{e.message}".brown
+  puts
+end
+
+puts " >> 4. Request to http://example.com - success".cyan
+Thread.current[:current_semian_sub_resource_name] = "sub_resource_2"
+response = Net::HTTP.get_response(uri)
+puts "  > Response status: #{response.code}"
+puts
+
+puts "> Review semian state:".blue.bold
+resource1 = Semian["nethttp_example_com_sub_resource_1"]
+puts "resource_name=#{resource1.name} resource=#{resource1} " \
+  "closed=#{resource1.closed?} open=#{resource1.open?} " \
+  "half_open=#{resource1.half_open?}".gray
+resource2 = Semian["nethttp_example_com_sub_resource_2"]
+puts "resource_name=#{resource2.name} resource=#{resource2} " \
+  "closed=#{resource2.closed?} open=#{resource2.open?} " \
+  "half_open=#{resource2.half_open?}".gray
+puts
+
+puts " >> 5. Request to http://example.com - fail".magenta
+begin
+  Thread.current[:current_semian_sub_resource_name] = "sub_resource_1"
+  Net::HTTP.get_response(uri)
+rescue Net::CircuitOpenError => e
+  puts "   >> Semian is open: #{e.message}".brown
+  puts "   !!! Semian open for sub_resource_1 and no request made to example.com:80 !!!".red.bold
+end
+puts
+
+puts " >> 6. Request to http://example.com - success".cyan
+Thread.current[:current_semian_sub_resource_name] = "sub_resource_2"
+response = Net::HTTP.get_response(uri)
+puts "  > Response status: #{response.code}"
+puts
+
+puts "> That's all Folks!".green

lib/semian/adapter.rb

@@ -9,19 +9,25 @@ def semian_identifier
     end
 
     def semian_resource
-      @semian_resource ||= case semian_options
+      return @semian_resource if @semian_resource
+
+      case semian_options
       when false
-        UnprotectedResource.new(semian_identifier)
+        @semian_resource = UnprotectedResource.new(semian_identifier)
       when nil
         Semian.logger.info("Semian is not configured for #{self.class.name}: #{semian_identifier}")
-        UnprotectedResource.new(semian_identifier)
+        @semian_resource = UnprotectedResource.new(semian_identifier)
       else
         options = semian_options.dup
         options.delete(:name)
         options[:consumer] = self
         options[:exceptions] ||= []
         options[:exceptions] += resource_exceptions
-        ::Semian.retrieve_or_register(semian_identifier, **options)
+        resource = ::Semian.retrieve_or_register(semian_identifier, **options)
+
+        @semian_resource = resource unless options.fetch(:dynamic, false)
+
+        resource
       end
     end
 
@@ -53,7 +59,11 @@ def semian_options
       return @semian_options if defined? @semian_options
 
       options = raw_semian_options
-      @semian_options = options && options.map { |k, v| [k.to_sym, v] }.to_h
+
+      symbolized_options = options && options.transform_keys(&:to_sym) # rubocop:disable Style/SafeNavigation
+      symbolized_options.tap do
+        @semian_options = symbolized_options if !symbolized_options || !symbolized_options.fetch(:dynamic, false)
+      end
     end
 
     def raw_semian_options

lib/semian/net_http.rb

@@ -91,16 +91,20 @@ def disabled?
     end
 
     def connect
-      return super if disabled?
+      with_cleared_dynamic_options do
+        return super if disabled?
 
-      acquire_semian_resource(adapter: :http, scope: :connection) { super }
+        acquire_semian_resource(adapter: :http, scope: :connection) { super }
+      end
     end
 
     def transport_request(*)
-      return super if disabled?
+      with_cleared_dynamic_options do
+        return super if disabled?
 
-      acquire_semian_resource(adapter: :http, scope: :query) do
-        handle_error_responses(super)
+        acquire_semian_resource(adapter: :http, scope: :query) do
+          handle_error_responses(super)
+        end
       end
     end
 
@@ -125,6 +129,24 @@ def handle_error_responses(result)
       end
       result
     end
+
+    def with_cleared_dynamic_options
+      unless @resource_acquisition_in_progress
+        @resource_acquisition_in_progress = true
+        resource_acquisition_started = true
+      end
+
+      yield
+    ensure
+      if resource_acquisition_started
+        if @raw_semian_options&.fetch(:dynamic, false)
+          # Clear @raw_semian_options if the resource was flagged as dynamic.
+          @raw_semian_options = nil
+        end
+
+        @resource_acquisition_in_progress = false
+      end
+    end
   end
 end
 

test/adapter_test.rb

@@ -91,6 +91,15 @@ def test_consumer_registration_does_not_prevent_gc
     end
   end
 
+  def test_does_not_memoize_dynamic_options
+    dynamic_client = Semian::DynamicAdapterTestClient.new(quota: 0.5)
+
+    refute_nil(dynamic_client.semian_resource)
+    assert_equal(4, dynamic_client.raw_semian_options[:success_threshold])
+    assert_equal(5, dynamic_client.raw_semian_options[:success_threshold])
+    assert_nil(dynamic_client.instance_variable_get("@semian_options"))
+  end
+
   class MyAdapterError < StandardError
     include Semian::AdapterError
   end

test/adapters/net_http_test.rb

@@ -23,6 +23,10 @@ def setup
     destroy_all_semian_resources
   end
 
+  def teardown
+    Thread.current[:sub_resource_name] = nil
+  end
+
   def test_semian_identifier
     with_server do
       with_semian_configuration do
@@ -484,6 +488,31 @@ def test_semian_disabled_env
     ENV.delete("SEMIAN_DISABLED")
   end
 
+  def test_dynamic_config
+    options = proc do |host, port|
+      DEFAULT_SEMIAN_OPTIONS.merge(
+        dynamic: true,
+        name: "#{host}_#{port}_#{Thread.current[:sub_resource_name]}",
+      )
+    end
+    host = SemianConfig["toxiproxy_upstream_host"]
+    port = SemianConfig["http_toxiproxy_port"]
+    http = Net::HTTP.new(host, port)
+    Thread.current[:sub_resource_name] = "service_a"
+
+    with_semian_configuration(options) do
+      with_server do
+        http.get("/200")
+
+        Thread.current[:sub_resource_name] = "service_b"
+        http.get("/200")
+
+        assert_includes(Semian.resources.keys, "nethttp_#{host}_#{port}_service_a")
+        assert_includes(Semian.resources.keys, "nethttp_#{host}_#{port}_service_b")
+      end
+    end
+  end
+
   private
 
   def half_open_cicuit!(backwards_time_travel = 10)

test/helpers/adapter_helper.rb

@@ -19,6 +19,28 @@ def resource_exceptions
     end
   end
 
+  module DynamicAdapterTest
+    include Semian::Adapter
+
+    def semian_identifier
+      :dynamic_semian_adapter_test
+    end
+
+    def raw_semian_options
+      {
+        bulkhead: false,
+        error_threshold: 1,
+        error_timeout: 1,
+        dynamic: true,
+        success_threshold: @current_success_threshold += 1,
+      }
+    end
+
+    def resource_exceptions
+      []
+    end
+  end
+
   class AdapterTestClient
     include AdapterTest
 
@@ -34,4 +56,16 @@ def ==(other)
       inspect == other.inspect
     end
   end
+
+  class DynamicAdapterTestClient
+    include DynamicAdapterTest
+
+    def initialize(**args)
+      @current_success_threshold = 1
+    end
+
+    def ==(other)
+      inspect == other.inspect
+    end
+  end
 end