diff --git a/CHANGELOG.md b/CHANGELOG.md index aec20f71..9db23175 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 0.9.2 (Next) +* [#167](https://github.com/slack-ruby/slack-ruby-client/pull/167): Added support for pausing between paginated requests that can cause Slack rate limiting - [@jmanian](https://github.com/jmanian). * [#163](https://github.com/slack-ruby/slack-ruby-client/pull/164): Use `OpenSSL::X509::DEFAULT_CERT_DIR` and `OpenSSL::X509::DEFAULT_CERT_FILE` for default ca_cert and ca_file - [@leifcr](https://github.com/leifcr). * [#161](https://github.com/slack-ruby/slack-ruby-client/pull/161): Added support for cursor pagination - [@dblock](https://github.com/dblock). * Your contribution here. diff --git a/README.md b/README.md index 64ca935e..e8d38bdc 100644 --- a/README.md +++ b/README.md @@ -164,18 +164,19 @@ client = Slack::Web::Client.new(user_agent: 'Slack Ruby Client/1.0') The following settings are supported. -setting | description -------------------|------------------------------------------------------------------------------------------------- -token | Slack API token. -user_agent | User-agent, defaults to _Slack Ruby Client/version_. -proxy | Optional HTTP proxy. -ca_path | Optional SSL certificates path. -ca_file | Optional SSL certificates file. -endpoint | Slack endpoint, default is _https://slack.com/api_. -logger | Optional `Logger` instance that logs HTTP requests. -timeout | Optional open/read timeout in seconds. -open_timeout | Optional connection open timeout in seconds. -default_page_size | Optional page size for paginated requests, default is _100_. +setting | description +--------------------|------------------------------------------------------------------------------------------------- +token | Slack API token. +user_agent | User-agent, defaults to _Slack Ruby Client/version_. +proxy | Optional HTTP proxy. +ca_path | Optional SSL certificates path. +ca_file | Optional SSL certificates file. +endpoint | Slack endpoint, default is _https://slack.com/api_. +logger | Optional `Logger` instance that logs HTTP requests. +timeout | Optional open/read timeout in seconds. +open_timeout | Optional connection open timeout in seconds. +default_page_size | Optional page size for paginated requests, default is _100_. +default_max_retries | Optional number of retries for paginated requests, default is _100_. You can also pass request options, including `timeout` and `open_timeout` into individual calls. @@ -195,6 +196,20 @@ end all_members # many thousands of team members retrieved 10 at a time ``` +When using cursor pagination the client will automatically pause and then retry the request if it runs into Slack rate limiting. (It will pause according to the `Retry-After` header in the 429 response before retrying the request.) If it receives too many rate-limited responses in a row it will give up and raise an error. The default number of retries is 100 and can be adjusted via `Slack::Web::Client.config.default_max_retries` or by passing it directly into the method as `max_retries`. + +You can also proactively avoid rate limiting by adding a pause between every paginated request with the `sleep_interval` parameter, which is given in seconds. + +```ruby +all_members = [] +client.users_list(presence: true, limit: 10, sleep_interval: 5, max_retries: 20) do |response| + # pauses for 5 seconds between each request + # gives up after 20 consecutive rate-limited responses + all_members.concat(response.members) +end +all_members # many thousands of team members retrieved 10 at a time +``` + ### RealTime Client The Real Time Messaging API is a WebSocket-based API that allows you to receive events from Slack in real time and send messages as user. diff --git a/lib/slack/web/config.rb b/lib/slack/web/config.rb index 0b5ca6d1..584e38ec 100644 --- a/lib/slack/web/config.rb +++ b/lib/slack/web/config.rb @@ -13,7 +13,8 @@ module Config :token, :timeout, :open_timeout, - :default_page_size + :default_page_size, + :default_max_retries ].freeze attr_accessor(*Config::ATTRIBUTES) @@ -29,6 +30,7 @@ def reset self.timeout = nil self.open_timeout = nil self.default_page_size = 100 + self.default_max_retries = 100 end end diff --git a/lib/slack/web/pagination/cursor.rb b/lib/slack/web/pagination/cursor.rb index 5f4fedaf..cbc4f350 100644 --- a/lib/slack/web/pagination/cursor.rb +++ b/lib/slack/web/pagination/cursor.rb @@ -7,23 +7,38 @@ class Cursor attr_reader :client attr_reader :verb + attr_reader :sleep_interval + attr_reader :max_retries attr_reader :params def initialize(client, verb, params = {}) @client = client @verb = verb - @params = params + @sleep_interval = params[:sleep_interval] + @max_retries = params[:max_retries] || client.default_max_retries + @params = params.reject { |k, _| [:sleep_interval, :max_retries].include?(k) } end def each next_cursor = nil + retry_count = 0 loop do query = { limit: client.default_page_size }.merge(params).merge(cursor: next_cursor) - response = client.send(verb, query) + begin + response = client.send(verb, query) + rescue Slack::Web::Api::Errors::TooManyRequestsError => e + raise e if retry_count >= max_retries + client.logger.debug("#{self.class}##{__method__}") { e.to_s } + retry_count += 1 + sleep(e.retry_after.seconds) + next + end yield response break unless response.response_metadata next_cursor = response.response_metadata.next_cursor break if next_cursor.blank? + retry_count = 0 + sleep(sleep_interval) if sleep_interval end end end diff --git a/spec/slack/web/api/pagination/cursor_spec.rb b/spec/slack/web/api/pagination/cursor_spec.rb index 2d24f3c7..efcc0595 100644 --- a/spec/slack/web/api/pagination/cursor_spec.rb +++ b/spec/slack/web/api/pagination/cursor_spec.rb @@ -21,8 +21,32 @@ Slack::Messages::Message.new(response_metadata: { next_cursor: 'next' }), Slack::Messages::Message.new ) + expect(cursor).not_to receive(:sleep) cursor.to_a end + context 'with rate limiting' do + let(:error) { Slack::Web::Api::Errors::TooManyRequestsError.new(nil) } + context 'with default max retries' do + it 'sleeps after a TooManyRequestsError' do + expect(client).to receive(:users_list).with(limit: 100, cursor: nil).ordered.and_return(Slack::Messages::Message.new(response_metadata: { next_cursor: 'next' })) + expect(client).to receive(:users_list).with(limit: 100, cursor: 'next').ordered.and_raise(error) + expect(error).to receive(:retry_after).once.ordered.and_return(9) + expect(cursor).to receive(:sleep).once.ordered.with(9) + expect(client).to receive(:users_list).with(limit: 100, cursor: 'next').ordered.and_return(Slack::Messages::Message.new) + cursor.to_a + end + end + context 'with a custom max_retries' do + let(:cursor) { Slack::Web::Api::Pagination::Cursor.new(client, 'users_list', max_retries: 4) } + it 'raises the error after hitting the max retries' do + expect(client).to receive(:users_list).with(limit: 100, cursor: nil).and_return(Slack::Messages::Message.new(response_metadata: { next_cursor: 'next' })) + expect(client).to receive(:users_list).with(limit: 100, cursor: 'next').exactly(5).times.and_raise(error) + expect(error).to receive(:retry_after).exactly(4).times.and_return(9) + expect(cursor).to receive(:sleep).exactly(4).times.with(9) + expect { cursor.to_a }.to raise_error(error) + end + end + end end context 'with a custom limit' do let(:cursor) { Slack::Web::Api::Pagination::Cursor.new(client, 'users_list', limit: 42) } @@ -31,4 +55,16 @@ cursor.first end end + context 'with a custom sleep_interval' do + let(:cursor) { Slack::Web::Api::Pagination::Cursor.new(client, 'users_list', sleep_interval: 3) } + it 'sleeps between requests' do + expect(client).to receive(:users_list).exactly(3).times.and_return( + Slack::Messages::Message.new(response_metadata: { next_cursor: 'next_a' }), + Slack::Messages::Message.new(response_metadata: { next_cursor: 'next_b' }), + Slack::Messages::Message.new + ) + expect(cursor).to receive(:sleep).with(3).twice + cursor.to_a + end + end end