Merge pull request #190 from travis-ci/rkh-better-throttling

better request throttling and auto-banning

Gemfile.lock

@@ -50,7 +50,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-core.git
-  revision: 3bf0ef70894375578a9af55e0eb9566eb9424294
+  revision: d76e7aa7bac71737553fe127fe825981b8a2bbf6
   specs:
     travis-core (0.0.1)
       actionmailer (~> 3.2.19)
@@ -81,7 +81,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-support.git
-  revision: e7f81093f83bd029cca6508739c5720e57e3d571
+  revision: a56f184c84f1f243da40daeaaf07f4ba931e6384
   specs:
     travis-support (0.0.1)
 

lib/travis/api/app.rb

@@ -13,7 +13,7 @@ require 'rack/contrib'
 require 'dalli'
 require 'memcachier'
 require 'rack/cache'
-require 'rack/attack'
+require 'travis/api/attack'
 require 'active_record'
 require 'redis'
 require 'gh'
@@ -96,53 +96,6 @@ module Travis::Api
         use Travis::Api::App::Middleware::Skylight
         use(Rack::Config) { |env| env['metriks.request.start'] ||= Time.now.utc }
 
-        Rack::Utils::HTTP_STATUS_CODES[420] = "Enhance Your Calm"
-
-        use Rack::Attack
-        Rack::Attack.blacklist('block client requesting ruby builds') do |req|
-          Travis.redis.sismember(:api_blacklisted_ips, req.ip)
-        end
-
-        # Lockout IP addresses that are hammering /auth/github.
-        # After 10 requests in 1 minute, block all requests from that IP for 1 hour.
-        Rack::Attack.blacklist('allow2ban login scrapers') do |req|
-          # `filter` returns false value if request is to your login page (but still
-          # increments the count) so request below the limit are not blocked until
-          # they hit the limit.  At that point, filter will return true and block.
-          Rack::Attack::Allow2Ban.filter(req.ip, :maxretry => 10, :findtime => 1.minute, :bantime => 1.hour) do
-            # The count for the IP is incremented if the return value is truthy.
-            req.path == '/auth/github' and req.post?
-          end
-        end
-
-        Rack::Attack.blacklisted_response = lambda do |env|
-          [ 420, {}, ['Enhance Your Calm']]
-        end
-
-        Rack::Attack.throttle('req/ip/1min', limit: 100, period: 1.minutes) do |req|
-          req.ip
-        end
-
-        Rack::Attack.throttle('req/ip/5min', limit: 300, period: 5.minutes) do |req|
-          req.ip
-        end
-
-        Rack::Attack.throttle('req/ip/10min', limit: 1000, period: 10.minutes) do |req|
-          req.ip
-        end
-
-        if ENV["MEMCACHIER_SERVERS"]
-          Rack::Attack.cache.store = Dalli::Client.new(
-            ENV["MEMCACHIER_SERVERS"].split(","),
-            username:             ENV["MEMCACHIER_USERNAME"],
-            password:             ENV["MEMCACHIER_PASSWORD"],
-            failover:             true,
-            socket_timeout:       1.5,
-            socket_failure_delay: 0.2)
-        else
-          Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new
-        end
-
         use Travis::Api::App::Cors # if Travis.env == 'development' ???
         use Raven::Rack if Travis.env == 'production' || Travis.env == 'staging'
         use Rack::SSL if Endpoint.production?
@@ -171,6 +124,9 @@ module Travis::Api
         use Travis::Api::App::Middleware::UserAgentTracker
         use Travis::Api::App::Middleware::Metriks
 
+        # make sure this is below ScopeCheck so we have the token
+        use Rack::Attack if Endpoint.production?
+
         # if this is a v3 API request, ignore everything after
         use Travis::API::V3::OptIn
 

lib/travis/api/attack.rb

@@ -0,0 +1,63 @@
+require 'rack/attack'
+
+class Rack::Attack
+  class Request
+    TOKEN = 'travis.access_token'.freeze
+
+    def travis_token
+      env.fetch(TOKEN)
+    end
+
+    def authenticated?
+      env.include? TOKEN
+    end
+
+    def identifier
+      authenticated? ? travis_token.to_s : ip
+    end
+  end
+
+  ####
+  # Ban based on: IP address
+  # Ban time:     indefinite
+  # Ban after:    manually banned
+  blacklist('block client requesting from redis') do |request|
+    Travis.redis.sismember(:api_blacklisted_ips, request.ip)
+  end
+
+  ####
+  # Ban based on: IP address or access token
+  # Ban time:     1 hour
+  # Ban after:    10 POST requests within one minute to /auth/github
+  blacklist('hammering /auth/github') do |request|
+     Rack::Attack::Allow2Ban.filter(request.identifier, maxretry: 10, findtime: 1.minute, bantime: 1.hour) do
+       request.post? and request.path == '/auth/github'
+     end
+  end
+
+  ###
+  # Throttle:  unauthenticated requests - 50 per minute
+  # Scoped by: IP address
+  throttle('req/ip/1min', limit: 50, period: 1.minute) do |request|
+    request.ip unless request.authenticated?
+  end
+
+  ###
+  # Throttle:  authenticated requests - 100 per minute
+  # Scoped by: access token
+  throttle('req/token/1min', limit: 100, period: 1.minute) do |request|
+    request.identifier
+  end
+
+  if ENV["MEMCACHIER_SERVERS"]
+    cache.store = Dalli::Client.new(
+      ENV["MEMCACHIER_SERVERS"].split(","),
+      username:             ENV["MEMCACHIER_USERNAME"],
+      password:             ENV["MEMCACHIER_PASSWORD"],
+      failover:             true,
+      socket_timeout:       1.5,
+      socket_failure_delay: 0.2)
+  else
+    cache.store = ActiveSupport::Cache::MemoryStore.new
+  end
+end