Handle HTTP 429 status gracefully

qa/qa/support/api.rb

@@ -8,9 +8,11 @@ module QA
       HTTP_STATUS_NO_CONTENT = 204
       HTTP_STATUS_ACCEPTED = 202
       HTTP_STATUS_NOT_FOUND = 404
+      HTTP_STATUS_TOO_MANY_REQUESTS = 429
       HTTP_STATUS_SERVER_ERROR = 500
 
       def post(url, payload, args = {})
+        with_retry_on_too_many_requests do
           default_args = {
             method: :post,
             url: url,
@@ -24,8 +26,10 @@ module QA
         rescue RestClient::ExceptionWithResponse => e
           return_response_or_raise(e)
         end
+      end
 
       def get(url, args = {})
+        with_retry_on_too_many_requests do
           default_args = {
             method: :get,
             url: url,
@@ -38,8 +42,10 @@ module QA
         rescue RestClient::ExceptionWithResponse => e
           return_response_or_raise(e)
         end
+      end
 
       def put(url, payload = nil)
+        with_retry_on_too_many_requests do
           RestClient::Request.execute(
             method: :put,
             url: url,
@@ -48,8 +54,10 @@ module QA
         rescue RestClient::ExceptionWithResponse => e
           return_response_or_raise(e)
         end
+      end
 
       def delete(url)
+        with_retry_on_too_many_requests do
           RestClient::Request.execute(
             method: :delete,
             url: url,
@@ -57,8 +65,10 @@ module QA
         rescue RestClient::ExceptionWithResponse => e
           return_response_or_raise(e)
         end
+      end
 
       def head(url)
+        with_retry_on_too_many_requests do
           RestClient::Request.execute(
             method: :head,
             url: url,
@@ -66,6 +76,26 @@ module QA
         rescue RestClient::ExceptionWithResponse => e
           return_response_or_raise(e)
         end
+      end
+
+      def with_retry_on_too_many_requests
+        response = nil
+
+        Support::Retrier.retry_until do
+          response = yield
+
+          if response.code == HTTP_STATUS_TOO_MANY_REQUESTS
+            wait_seconds = response.headers[:retry_after].to_i
+            QA::Runtime::Logger.debug("Received 429 - Too many requests. Waiting for #{wait_seconds} seconds.")
+
+            sleep wait_seconds
+          end
+
+          response.code != HTTP_STATUS_TOO_MANY_REQUESTS
+        end
+
+        response
+      end
 
       def parse_body(response)
         JSON.parse(response.body, symbolize_names: true)