From 9303a24595dcefffce78778aadca223655e4ea43 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 17 Sep 2015 14:57:44 +0200 Subject: [PATCH 1/6] base throttling on access token if the call is authenticated, rather than on IP address, improve throttling rules --- lib/travis/api/app.rb | 52 +++-------------------------- lib/travis/api/attack.rb | 70 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 48 deletions(-) create mode 100644 lib/travis/api/attack.rb diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index 76a2795b..a5b1a37a 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -13,7 +13,7 @@ require 'rack/contrib' require 'dalli' require 'memcachier' require 'rack/cache' -require 'rack/attack' +require 'travis/api/attack' require 'active_record' require 'redis' require 'gh' @@ -96,53 +96,6 @@ module Travis::Api use Travis::Api::App::Middleware::Skylight use(Rack::Config) { |env| env['metriks.request.start'] ||= Time.now.utc } - Rack::Utils::HTTP_STATUS_CODES[420] = "Enhance Your Calm" - - use Rack::Attack - Rack::Attack.blacklist('block client requesting ruby builds') do |req| - Travis.redis.sismember(:api_blacklisted_ips, req.ip) - end - - # Lockout IP addresses that are hammering /auth/github. - # After 10 requests in 1 minute, block all requests from that IP for 1 hour. - Rack::Attack.blacklist('allow2ban login scrapers') do |req| - # `filter` returns false value if request is to your login page (but still - # increments the count) so request below the limit are not blocked until - # they hit the limit. At that point, filter will return true and block. - Rack::Attack::Allow2Ban.filter(req.ip, :maxretry => 10, :findtime => 1.minute, :bantime => 1.hour) do - # The count for the IP is incremented if the return value is truthy. - req.path == '/auth/github' and req.post? - end - end - - Rack::Attack.blacklisted_response = lambda do |env| - [ 420, {}, ['Enhance Your Calm']] - end - - Rack::Attack.throttle('req/ip/1min', limit: 100, period: 1.minutes) do |req| - req.ip - end - - Rack::Attack.throttle('req/ip/5min', limit: 300, period: 5.minutes) do |req| - req.ip - end - - Rack::Attack.throttle('req/ip/10min', limit: 1000, period: 10.minutes) do |req| - req.ip - end - - if ENV["MEMCACHIER_SERVERS"] - Rack::Attack.cache.store = Dalli::Client.new( - ENV["MEMCACHIER_SERVERS"].split(","), - username: ENV["MEMCACHIER_USERNAME"], - password: ENV["MEMCACHIER_PASSWORD"], - failover: true, - socket_timeout: 1.5, - socket_failure_delay: 0.2) - else - Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new - end - use Travis::Api::App::Cors # if Travis.env == 'development' ??? use Raven::Rack if Travis.env == 'production' || Travis.env == 'staging' use Rack::SSL if Endpoint.production? @@ -171,6 +124,9 @@ module Travis::Api use Travis::Api::App::Middleware::UserAgentTracker use Travis::Api::App::Middleware::Metriks + # make sure this is below ScopeCheck so we have the token + use Travis::Api::Attack + # if this is a v3 API request, ignore everything after use Travis::API::V3::OptIn diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb new file mode 100644 index 00000000..f61154c7 --- /dev/null +++ b/lib/travis/api/attack.rb @@ -0,0 +1,70 @@ +require 'rack/attack' + +module Travis::Api + class Attack < Rack::Attack + module Request + TOKEN = 'travis.access_token'.freeze + Rack::Attack::Request.prepend(self) + + def travis_token + env.fetch(TOKEN) + end + + def authenticated? + env.include? TOKEN + end + + def identifier + authenticated? ? travis_token.to_s : ip + end + end + + def self.cache + Rack::Attack.cache + end + + #### + # Ban based on: IP address + # Ban time: indefinite + # Ban after: manually banned + blacklist('block client requesting from redis') do |request| + Travis.redis.sismember(:api_blacklisted_ips, request.ip) + end + + #### + # Ban based on: IP address or access token + # Ban time: 1 hour + # Ban after: 10 POST requests within one minute to /auth/github + blacklist('hammering /auth/github') do |request| + Rack::Attack::Allow2Ban.filter(request.identifier, maxretry: 10, findtime: 1.minute, bantime: 1.hour) do + request.post? and request.path == '/auth/github' + end + end + + ### + # Throttle: unauthenticated requests - 50 per minute + # Scoped by: IP address + throttle('req/ip/1min', limit: 50, period: 1.minute) do |request| + request.ip unless request.authenticated? + end + + ### + # Throttle: authenticated requests - 100 per minute + # Scoped by: access token + throttle('req/token/1min', limit: 100, period: 1.minute) do |request| + request.identifier + end + + if ENV["MEMCACHIER_SERVERS"] + cache.store = Dalli::Client.new( + ENV["MEMCACHIER_SERVERS"].split(","), + username: ENV["MEMCACHIER_USERNAME"], + password: ENV["MEMCACHIER_PASSWORD"], + failover: true, + socket_timeout: 1.5, + socket_failure_delay: 0.2) + else + cache.store = ActiveSupport::Cache::MemoryStore.new + end + end +end From dc0da3645a6685c39cda8c73d591552b6532c72f Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 17 Sep 2015 15:10:27 +0200 Subject: [PATCH 2/6] work around strange constant lookup --- lib/travis/api/attack.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index f61154c7..47295c9d 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -2,6 +2,8 @@ require 'rack/attack' module Travis::Api class Attack < Rack::Attack + DalliProxy = Rack::Attack::DalliProxy # ? + module Request TOKEN = 'travis.access_token'.freeze Rack::Attack::Request.prepend(self) From e478c621f282d6cad80c69c846c632a0a037f4a8 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 17 Sep 2015 15:13:33 +0200 Subject: [PATCH 3/6] no more inheritance --- lib/travis/api/app.rb | 2 +- lib/travis/api/attack.rb | 120 +++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 63 deletions(-) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index a5b1a37a..7fa44e56 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -125,7 +125,7 @@ module Travis::Api use Travis::Api::App::Middleware::Metriks # make sure this is below ScopeCheck so we have the token - use Travis::Api::Attack + use Rack::Attack # if this is a v3 API request, ignore everything after use Travis::API::V3::OptIn diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index 47295c9d..bf82058b 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -1,72 +1,68 @@ require 'rack/attack' -module Travis::Api - class Attack < Rack::Attack - DalliProxy = Rack::Attack::DalliProxy # ? +class Rack::Attack + module RequestMixin + TOKEN = 'travis.access_token'.freeze + Rack::Attack::Request.prepend(self) - module Request - TOKEN = 'travis.access_token'.freeze - Rack::Attack::Request.prepend(self) - - def travis_token - env.fetch(TOKEN) - end - - def authenticated? - env.include? TOKEN - end - - def identifier - authenticated? ? travis_token.to_s : ip - end + def travis_token + env.fetch(TOKEN) end - def self.cache - Rack::Attack.cache + def authenticated? + env.include? TOKEN end - #### - # Ban based on: IP address - # Ban time: indefinite - # Ban after: manually banned - blacklist('block client requesting from redis') do |request| - Travis.redis.sismember(:api_blacklisted_ips, request.ip) - end - - #### - # Ban based on: IP address or access token - # Ban time: 1 hour - # Ban after: 10 POST requests within one minute to /auth/github - blacklist('hammering /auth/github') do |request| - Rack::Attack::Allow2Ban.filter(request.identifier, maxretry: 10, findtime: 1.minute, bantime: 1.hour) do - request.post? and request.path == '/auth/github' - end - end - - ### - # Throttle: unauthenticated requests - 50 per minute - # Scoped by: IP address - throttle('req/ip/1min', limit: 50, period: 1.minute) do |request| - request.ip unless request.authenticated? - end - - ### - # Throttle: authenticated requests - 100 per minute - # Scoped by: access token - throttle('req/token/1min', limit: 100, period: 1.minute) do |request| - request.identifier - end - - if ENV["MEMCACHIER_SERVERS"] - cache.store = Dalli::Client.new( - ENV["MEMCACHIER_SERVERS"].split(","), - username: ENV["MEMCACHIER_USERNAME"], - password: ENV["MEMCACHIER_PASSWORD"], - failover: true, - socket_timeout: 1.5, - socket_failure_delay: 0.2) - else - cache.store = ActiveSupport::Cache::MemoryStore.new + def identifier + authenticated? ? travis_token.to_s : ip end end + + def self.cache + Rack::Attack.cache + end + + #### + # Ban based on: IP address + # Ban time: indefinite + # Ban after: manually banned + blacklist('block client requesting from redis') do |request| + Travis.redis.sismember(:api_blacklisted_ips, request.ip) + end + + #### + # Ban based on: IP address or access token + # Ban time: 1 hour + # Ban after: 10 POST requests within one minute to /auth/github + blacklist('hammering /auth/github') do |request| + Rack::Attack::Allow2Ban.filter(request.identifier, maxretry: 10, findtime: 1.minute, bantime: 1.hour) do + request.post? and request.path == '/auth/github' + end + end + + ### + # Throttle: unauthenticated requests - 50 per minute + # Scoped by: IP address + throttle('req/ip/1min', limit: 50, period: 1.minute) do |request| + request.ip unless request.authenticated? + end + + ### + # Throttle: authenticated requests - 100 per minute + # Scoped by: access token + throttle('req/token/1min', limit: 100, period: 1.minute) do |request| + request.identifier + end + + if ENV["MEMCACHIER_SERVERS"] + cache.store = Dalli::Client.new( + ENV["MEMCACHIER_SERVERS"].split(","), + username: ENV["MEMCACHIER_USERNAME"], + password: ENV["MEMCACHIER_PASSWORD"], + failover: true, + socket_timeout: 1.5, + socket_failure_delay: 0.2) + else + cache.store = ActiveSupport::Cache::MemoryStore.new + end end From 5e40f33fc14ebdacf0b3cdaa961cec43cc360049 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 17 Sep 2015 15:18:48 +0200 Subject: [PATCH 4/6] remove left-overs from inheriting from Rack::Attack --- Gemfile.lock | 4 ++-- lib/travis/api/attack.rb | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 982fce5f..1f0bb5af 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 3bf0ef70894375578a9af55e0eb9566eb9424294 + revision: d76e7aa7bac71737553fe127fe825981b8a2bbf6 specs: travis-core (0.0.1) actionmailer (~> 3.2.19) @@ -81,7 +81,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-support.git - revision: e7f81093f83bd029cca6508739c5720e57e3d571 + revision: a56f184c84f1f243da40daeaaf07f4ba931e6384 specs: travis-support (0.0.1) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index bf82058b..4ad4aa48 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -1,10 +1,7 @@ require 'rack/attack' class Rack::Attack - module RequestMixin - TOKEN = 'travis.access_token'.freeze - Rack::Attack::Request.prepend(self) - + class Request def travis_token env.fetch(TOKEN) end @@ -18,10 +15,6 @@ class Rack::Attack end end - def self.cache - Rack::Attack.cache - end - #### # Ban based on: IP address # Ban time: indefinite From e8769dddc58a8297fbdcc2be20d354c1a54c3f95 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 17 Sep 2015 15:21:16 +0200 Subject: [PATCH 5/6] add missing constant --- lib/travis/api/attack.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index 4ad4aa48..dfde4004 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -2,6 +2,8 @@ require 'rack/attack' class Rack::Attack class Request + TOKEN = 'travis.access_token'.freeze + def travis_token env.fetch(TOKEN) end From c372b0734483329b76d429699391600f76f0b51d Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 17 Sep 2015 15:26:30 +0200 Subject: [PATCH 6/6] only enable request throttling in production --- lib/travis/api/app.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index 7fa44e56..a2763fdd 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -125,7 +125,7 @@ module Travis::Api use Travis::Api::App::Middleware::Metriks # make sure this is below ScopeCheck so we have the token - use Rack::Attack + use Rack::Attack if Endpoint.production? # if this is a v3 API request, ignore everything after use Travis::API::V3::OptIn