Merge pull request #55 from ParentSquare/clear-circuits

Add a Faulty#clear method to reset all circuits

Author: justinhoward

.rubocop.yml

@@ -73,6 +73,7 @@ RSpec/VerifiedDoubleReference: { Enabled: true }
 
 Metrics/AbcSize: { Max: 40 }
 Metrics/BlockLength: { Enabled: false }
+Metrics/ClassLength: { Enabled: false }
 Metrics/CyclomaticComplexity: { Enabled: false }
 Metrics/MethodLength: { Max: 30 }
 Metrics/PerceivedComplexity: { Enabled: false }

README.md

@@ -90,6 +90,7 @@ Also see "Release It!: Design and Deploy Production-Ready Software" by
   + [Other Built-in Listeners](#other-built-in-listeners)
   + [Custom Listeners](#custom-listeners)
 * [Disabling Faulty Globally](#disabling-faulty-globally)
+* [Testing with Faulty](#testing-with-faulty)
 * [How it Works](#how-it-works)
   + [Caching](#caching)
   + [Fault Tolerance](#fault-tolerance)
@@ -1173,6 +1174,24 @@ not affect the stored state of circuits.
 Faulty will **still use the cache** even when disabled. If you also want to
 disable the cache, configure Faulty to use a `Faulty::Cache::Null` cache.
 
+## Testing with Faulty
+
+Depending on your application, you could choose to
+[disable Faulty globally](#disabling-faulty-globally), but sometimes you may
+want to test your application's behavior in a failure scenario.
+
+If you have such tests, you will want to prevent failures in one test from
+affecting other tests. To clear all circuit states between tests, use `#clear!`.
+For example, with rspec:
+
+```ruby
+RSpec.configure do |config|
+  config.after do
+    Faulty.clear!
+  end
+end
+```
+
 ## How it Works
 
 Faulty implements a version of circuit breakers inspired by "Release It!: Design

lib/faulty.rb

@@ -116,6 +116,7 @@ def circuit(name, **config, &block)
 
     # Get a list of all circuit names for the default instance
     #
+    # @see #list_circuits
     # @return [Array<String>] The circuit names
     def list_circuits
       options.storage.list
@@ -157,6 +158,14 @@ def enable!
     def disabled?
       @disabled == true
     end
+
+    # Reset all circuits for the default instance
+    #
+    # @see #clear
+    # @return [void]
+    def clear!
+      default.clear
+    end
   end
 
   attr_reader :options
@@ -255,6 +264,20 @@ def list_circuits
     options.storage.list
   end
 
+  # Reset all circuits
+  #
+  # Intended for use in tests. This can be expensive and is not appropriate
+  # to call in production code
+  #
+  # See the documentation for your chosen backend for specific semantics and
+  # safety concerns. For example, the Redis backend resets all circuits, but
+  # it does not clear the circuit list to maintain thread-safety.
+  #
+  # @return [void]
+  def clear!
+    options.storage.clear
+  end
+
   private
 
   # Get circuit options from the {Faulty} options

lib/faulty/circuit.rb

@@ -23,7 +23,7 @@ class Faulty
   # write your own code to periodically check how long it has been running.
   # If you're sure you want ruby's generic Timeout, you can apply it yourself
   # inside the circuit run block.
-  class Circuit # rubocop:disable Metrics/ClassLength
+  class Circuit
     CACHE_REFRESH_SUFFIX = '.faulty_refresh'
 
     attr_reader :name

lib/faulty/storage/circuit_proxy.rb

@@ -58,6 +58,7 @@ def initialize(storage, **options, &block)
         status
         history
         list
+        clear
       ].each do |method|
         define_method(method) do |*args|
           options.circuit.run { @storage.public_send(method, *args) }

lib/faulty/storage/fallback_chain.rb

@@ -159,6 +159,14 @@ def list
         end
       end
 
+      # Clears circuits in all storage backends
+      #
+      # @param (see Interface#clear)
+      # @return (see Interface#clear)
+      def clear
+        send_all(:clear)
+      end
+
       # This is fault tolerant if any of the available backends are fault tolerant
       #
       # @param (see Interface#fault_tolerant?)

lib/faulty/storage/fault_tolerant_proxy.rb

@@ -81,7 +81,14 @@ def self.wrap(storage, **options, &block)
       #   @see Interface#list
       #   @param (see Interface#list)
       #   @return (see Interface#list)
-      def_delegators :@storage, :lock, :unlock, :reset, :history, :list
+      #
+      # @!method clear
+      #   Clear is not called in normal operation, so it doesn't capture errors
+      #
+      #   @see Interface#list
+      #   @param (see Interface#list)
+      #   @return (see Interface#list)
+      def_delegators :@storage, :lock, :unlock, :reset, :history, :list, :clear
 
       # Get circuit options safely
       #

lib/faulty/storage/interface.rb

@@ -172,6 +172,17 @@ def list
         raise NotImplementedError
       end
 
+      # Reset all circuits
+      #
+      # Some implementions may clear circuits on a best-effort basis since
+      # all circuits may not be known.
+      #
+      # @raise NotImplementedError If the storage backend does not support clearing.
+      # @return [void]
+      def clear
+        raise NotImplementedError
+      end
+
       # Can this storage backend raise an error?
       #
       # If the storage backend returns false from this method, it will be wrapped

lib/faulty/storage/memory.rb

@@ -196,6 +196,13 @@ def list
         @circuits.keys
       end
 
+      # Clears all circuits
+      #
+      # @return [void]
+      def clear
+        @circuits.clear
+      end
+
       # Memory storage is fault-tolerant by default
       #
       # @return [true]

lib/faulty/storage/null.rb

@@ -79,6 +79,11 @@ def list
         []
       end
 
+      # @param (see Interface#clear)
+      # @return (see Interface#clear)
+      def clear
+      end
+
       # This backend is fault tolerant
       #
       # @param (see Interface#fault_tolerant?)

lib/faulty/storage/redis.rb

@@ -9,7 +9,7 @@ module Storage
     # cascading failures in your application when evaluating circuits. Always
     # wrap this backend with a {FaultTolerantProxy} to limit the effect of
     # these types of events.
-    class Redis # rubocop:disable Metrics/ClassLength
+    class Redis
       # Separates the time/status for history entry strings
       ENTRY_SEPARATOR = ':'
 
@@ -95,7 +95,7 @@ def initialize(**options, &block)
       # @param (see Interface#get_options)
       # @return (see Interface#get_options)
       def get_options(circuit)
-        json = redis { |r| r.get(options_key(circuit)) }
+        json = redis { |r| r.get(options_key(circuit.name)) }
         return if json.nil?
 
         JSON.parse(json, symbolize_names: true)
@@ -110,7 +110,7 @@ def get_options(circuit)
       # @return (see Interface#set_options)
       def set_options(circuit, stored_options)
         redis do |r|
-          r.set(options_key(circuit), JSON.dump(stored_options), ex: options.circuit_ttl)
+          r.set(options_key(circuit.name), JSON.dump(stored_options), ex: options.circuit_ttl)
         end
       end
 
@@ -120,7 +120,7 @@ def set_options(circuit, stored_options)
       # @param (see Interface#entry)
       # @return (see Interface#entry)
       def entry(circuit, time, success, status)
-        key = entries_key(circuit)
+        key = entries_key(circuit.name)
         result = pipe do |r|
           r.sadd(list_key, circuit.name)
           r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
@@ -139,11 +139,11 @@ def entry(circuit, time, success, status)
       # @param (see Interface#open)
       # @return (see Interface#open)
       def open(circuit, opened_at)
-        key = state_key(circuit)
+        key = state_key(circuit.name)
         ex = options.circuit_ttl
         result = watch_exec(key, ['closed', nil]) do |m|
           m.set(key, 'open', ex: ex)
-          m.set(opened_at_key(circuit), opened_at, ex: ex)
+          m.set(opened_at_key(circuit.name), opened_at, ex: ex)
         end
 
         result && result[0] == 'OK'
@@ -155,7 +155,7 @@ def open(circuit, opened_at)
       # @param (see Interface#reopen)
       # @return (see Interface#reopen)
       def reopen(circuit, opened_at, previous_opened_at)
-        key = opened_at_key(circuit)
+        key = opened_at_key(circuit.name)
         result = watch_exec(key, [previous_opened_at.to_s]) do |m|
           m.set(key, opened_at, ex: options.circuit_ttl)
         end
@@ -169,11 +169,11 @@ def reopen(circuit, opened_at, previous_opened_at)
       # @param (see Interface#close)
       # @return (see Interface#close)
       def close(circuit)
-        key = state_key(circuit)
+        key = state_key(circuit.name)
         ex = options.circuit_ttl
         result = watch_exec(key, ['open']) do |m|
           m.set(key, 'closed', ex: ex)
-          m.del(entries_key(circuit))
+          m.del(entries_key(circuit.name))
         end
 
         result && result[0] == 'OK'
@@ -187,7 +187,7 @@ def close(circuit)
       # @param (see Interface#lock)
       # @return (see Interface#lock)
       def lock(circuit, state)
-        redis { |r| r.set(lock_key(circuit), state) }
+        redis { |r| r.set(lock_key(circuit.name), state) }
       end
 
       # Unlock a circuit
@@ -196,7 +196,7 @@ def lock(circuit, state)
       # @param (see Interface#unlock)
       # @return (see Interface#unlock)
       def unlock(circuit)
-        redis { |r| r.del(lock_key(circuit)) }
+        redis { |r| r.del(lock_key(circuit.name)) }
       end
 
       # Reset a circuit
@@ -205,14 +205,15 @@ def unlock(circuit)
       # @param (see Interface#reset)
       # @return (see Interface#reset)
       def reset(circuit)
+        name = circuit.is_a?(Circuit) ? circuit.name : circuit
         pipe do |r|
           r.del(
-            entries_key(circuit),
-            opened_at_key(circuit),
-            lock_key(circuit),
-            options_key(circuit)
+            entries_key(name),
+            opened_at_key(name),
+            lock_key(name),
+            options_key(name)
           )
-          r.set(state_key(circuit), 'closed', ex: options.circuit_ttl)
+          r.set(state_key(name), 'closed', ex: options.circuit_ttl)
         end
       end
 
@@ -224,10 +225,10 @@ def reset(circuit)
       def status(circuit)
         futures = {}
         pipe do |r|
-          futures[:state] = r.get(state_key(circuit))
-          futures[:lock] = r.get(lock_key(circuit))
-          futures[:opened_at] = r.get(opened_at_key(circuit))
-          futures[:entries] = r.lrange(entries_key(circuit), 0, -1)
+          futures[:state] = r.get(state_key(circuit.name))
+          futures[:lock] = r.get(lock_key(circuit.name))
+          futures[:opened_at] = r.get(opened_at_key(circuit.name))
+          futures[:entries] = r.lrange(entries_key(circuit.name), 0, -1)
         end
 
         state = futures[:state].value&.to_sym || :closed
@@ -249,7 +250,7 @@ def status(circuit)
       # @param (see Interface#history)
       # @return (see Interface#history)
       def history(circuit)
-        entries = redis { |r| r.lrange(entries_key(circuit), 0, -1) }
+        entries = redis { |r| r.lrange(entries_key(circuit.name), 0, -1) }
         map_entries(entries).reverse
       end
 
@@ -260,6 +261,21 @@ def list
         redis { |r| r.sunion(*all_list_keys) }
       end
 
+      # Reset all circuits
+      #
+      # This does not empty the list of circuits as returned by {#list}. This is
+      # because that would be a thread-usafe operation that could result in
+      # circuits not being in the list.
+      #
+      # This implmenentation resets circuits individually, and will be very
+      # slow for large numbers of circuits. It should not be used in production
+      # code.
+      #
+      # @return [void]
+      def clear
+        list.each { |c| reset(c) }
+      end
+
       # Redis storage is not fault-tolerant
       #
       # @return [true]
@@ -276,33 +292,33 @@ def key(*parts)
         [options.key_prefix, *parts].join(options.key_separator)
       end
 
-      def ckey(circuit, *parts)
-        key('circuit', circuit.name, *parts)
+      def ckey(circuit_name, *parts)
+        key('circuit', circuit_name, *parts)
       end
 
       # @return [String] The key for circuit options
-      def options_key(circuit)
-        ckey(circuit, 'options')
+      def options_key(circuit_name)
+        ckey(circuit_name, 'options')
       end
 
       # @return [String] The key for circuit state
-      def state_key(circuit)
-        ckey(circuit, 'state')
+      def state_key(circuit_name)
+        ckey(circuit_name, 'state')
       end
 
       # @return [String] The key for circuit run history entries
-      def entries_key(circuit)
-        ckey(circuit, 'entries')
+      def entries_key(circuit_name)
+        ckey(circuit_name, 'entries')
       end
 
       # @return [String] The key for circuit locks
-      def lock_key(circuit)
-        ckey(circuit, 'lock')
+      def lock_key(circuit_name)
+        ckey(circuit_name, 'lock')
       end
 
       # @return [String] The key for circuit opened_at
-      def opened_at_key(circuit)
-        ckey(circuit, 'opened_at')
+      def opened_at_key(circuit_name)
+        ckey(circuit_name, 'opened_at')
       end
 
       # Get the current key to add circuit names to

spec/faulty_spec.rb

@@ -179,4 +179,10 @@
     described_class.enable!
     expect(described_class.disabled?).to be(false)
   end
+
+  it 'clears circuits' do
+    instance.circuit('test').run { 'ok' }
+    instance.clear!
+    expect(instance.circuit('test').history).to eq([])
+  end
 end