From 7e438be4cfa324b15c98222559705a86bd3bf2ec Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 16 Mar 2016 15:20:41 +0100 Subject: [PATCH 1/8] remove hard coded request path --- lib/travis/api/attack.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index 207396f5..17fcd44a 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -61,7 +61,7 @@ class Rack::Attack # Ban after: 10 POST requests within 30 seconds blacklist('spamming with POST requests') do |request| Rack::Attack::Allow2Ban.filter(request.identifier, maxretry: 10, findtime: 30.seconds, bantime: bantime(1.hour)) do - request.post? and not POST_WHITELISTED.include? '/auth/github' + request.post? and not POST_WHITELISTED.include? request.path end end From b68c7a33b03b9f8bb937aafbb9b3b4b10d5072cf Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 10:57:12 +0100 Subject: [PATCH 2/8] Allow to set sidekiq concurrency separately from web workers Unicorn, which we use to serve the API, uses forked workers that need only one connection per instance. Sidekiq on the other hand runs several instances in memory, so it needs a higher concurrency setting. This commit introduces a way to set sidekiq db pool using SIDEKIQ_DB_POOL_SIZE --- lib/travis/sidekiq.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index ef5aa91c..38b651be 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -7,6 +7,7 @@ require 'travis/api/workers/job_cancellation' require 'travis/api/workers/job_restart' require 'travis/support/amqp' +Travis.config.database[:pool] = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 Travis::Database.connect if Travis.config.logs_database From 5682502cbaa8d35a751f6179971e77a5c0cf1af4 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:14:46 +0100 Subject: [PATCH 3/8] Bump travis-core --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6701537d..3553f293 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 5bff6bf138d31754e38b8c148e56ba8ceab39859 + revision: 808cc8f32e6e1054e7231c14a36b0489b09446ff specs: travis-core (0.0.1) actionmailer (~> 3.2.19) @@ -331,7 +331,7 @@ GEM treetop (1.4.15) polyglot polyglot (>= 0.3.1) - tzinfo (0.3.46) + tzinfo (0.3.47) unicorn (4.8.3) kgio (~> 2.6) rack @@ -390,4 +390,4 @@ DEPENDENCIES yard-sinatra! BUNDLED WITH - 1.10.6 + 1.11.2 From f2c96b3eb52cede315acde77743b924106e05195 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:27:42 +0100 Subject: [PATCH 4/8] Set sidekiq pool size also for logs DB --- lib/travis/sidekiq.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index 38b651be..936c4653 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -7,7 +7,9 @@ require 'travis/api/workers/job_cancellation' require 'travis/api/workers/job_restart' require 'travis/support/amqp' -Travis.config.database[:pool] = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 +pool_size = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 +Travis.config.database[:pool] = pool_size +Travis.config.logs_database[:pool] = pool_size Travis::Database.connect if Travis.config.logs_database From 6aaa25a099a457e96001701460578777fb45b22a Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:45:46 +0100 Subject: [PATCH 5/8] Overwrite database pool size for logs database Logs database config is set in travis keychain and is set to a concurrency much higher than what we set in heroku config. I don't want to change it globally, because it may need to be different for other apps, so I'm changing it here just before we connect to the db. --- lib/travis/api/app.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index edc64310..25af74ba 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -199,6 +199,9 @@ module Travis::Api Travis::Database.connect if Travis.config.logs_database + pool_size = ENV['DATABASE_POOL_SIZE'] + Travis.config.logs_database[:pool] = pool_size if pool_size + Log.establish_connection 'logs_database' Log::Part.establish_connection 'logs_database' end From f0dcf270bc44cb8256d99ba438793dc05352b2f9 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:59:34 +0100 Subject: [PATCH 6/8] Always convert pool size to integer --- lib/travis/api/app.rb | 2 +- lib/travis/sidekiq.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index 25af74ba..d9719090 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -200,7 +200,7 @@ module Travis::Api if Travis.config.logs_database pool_size = ENV['DATABASE_POOL_SIZE'] - Travis.config.logs_database[:pool] = pool_size if pool_size + Travis.config.logs_database[:pool] = pool_size.to_i if pool_size Log.establish_connection 'logs_database' Log::Part.establish_connection 'logs_database' diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index 936c4653..bb9e3574 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -8,8 +8,8 @@ require 'travis/api/workers/job_restart' require 'travis/support/amqp' pool_size = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 -Travis.config.database[:pool] = pool_size -Travis.config.logs_database[:pool] = pool_size +Travis.config.database[:pool] = pool_size.to_i +Travis.config.logs_database[:pool] = pool_size.to_i Travis::Database.connect if Travis.config.logs_database From 5294f1dab25523ca0db8debd8a9c3d5697d2d939 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 13:00:29 +0100 Subject: [PATCH 7/8] Bump travis-core --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3553f293..f4305853 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 808cc8f32e6e1054e7231c14a36b0489b09446ff + revision: f7b3a76b3f39c28bb5cf7b9dc24acec13908a11a specs: travis-core (0.0.1) actionmailer (~> 3.2.19) From 8a050a468c77f2ad8c5faa99864c0bd7d61b08a0 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 13:07:23 +0100 Subject: [PATCH 8/8] Use shared logs model to establish connection That way we will use only one connection to the logs database --- lib/travis/api/app.rb | 3 +-- lib/travis/sidekiq.rb | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index d9719090..6b72b8ce 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -202,8 +202,7 @@ module Travis::Api pool_size = ENV['DATABASE_POOL_SIZE'] Travis.config.logs_database[:pool] = pool_size.to_i if pool_size - Log.establish_connection 'logs_database' - Log::Part.establish_connection 'logs_database' + Travis::LogsModel.establish_connection 'logs_database' end end diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index bb9e3574..077e6055 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -13,8 +13,7 @@ Travis.config.logs_database[:pool] = pool_size.to_i Travis::Database.connect if Travis.config.logs_database - Log.establish_connection 'logs_database' - Log::Part.establish_connection 'logs_database' + Travis::LogsModel.establish_connection 'logs_database' end Travis::Async.enabled = true