From 759fdf203b0ee39e9d0e15d68ae904a96ace1b3f Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 11 Dec 2015 12:35:37 +0100 Subject: [PATCH 01/45] change rakefile and db congig --- Rakefile | 1 - config/database.yml | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Rakefile b/Rakefile index 866a2958..9a143017 100644 --- a/Rakefile +++ b/Rakefile @@ -8,7 +8,6 @@ begin rescue LoadError # we can't load micro migrations on production end -require 'travis' begin require 'rspec/core/rake_task' diff --git a/config/database.yml b/config/database.yml index 26b09327..53e59a34 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,9 +15,8 @@ production: development: <<: *defaults - database: travis_development + database: travis_pro_development test: <<: *defaults - database: travis_test - + database: travis_pro_test From efab4be6de172e4ba325f57454669da27fb16ded Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:03:09 +0100 Subject: [PATCH 02/45] comment out schema env definition --- Rakefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Rakefile b/Rakefile index 9a143017..58566cbf 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,13 @@ require 'bundler/setup' -require 'travis' +# require 'travis' require 'travis/engine' -begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production -end +# begin +# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) +# require 'micro_migrations' +# rescue LoadError +# # we can't load micro migrations on production +# end begin require 'rspec/core/rake_task' From 4833f4173214389419227674eddc9f8134c943ce Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:08:31 +0100 Subject: [PATCH 03/45] use sql schema --- Gemfile | 1 + Gemfile.lock | 10 ++++++++++ Rakefile | 17 ++++++++++------- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 9893ebf5..bf2e564b 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ group :test do gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' + gem 'travis-migrations', github: 'travis-ci/travis-migrations' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 09ee17b3..a340c150 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -72,6 +72,12 @@ GIT travis-config (~> 0.1.0) virtus (~> 1.0.0) +GIT + remote: git://github.com/travis-ci/travis-migrations.git + revision: d6105ad77303bcd04d1e376070a11dbc8b3077c7 + specs: + travis-migrations (0.0.1) + GIT remote: git://github.com/travis-ci/travis-sidekiqs.git revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23 @@ -376,8 +382,12 @@ DEPENDENCIES travis-api! travis-config (~> 0.1.0) travis-core! + travis-migrations! travis-sidekiqs! travis-support! travis-yaml! unicorn yard-sinatra! + +BUNDLED WITH + 1.10.6 diff --git a/Rakefile b/Rakefile index 58566cbf..016d761c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,16 @@ require 'bundler/setup' -# require 'travis' +require 'travis/migrations' require 'travis/engine' -# begin -# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) -# require 'micro_migrations' -# rescue LoadError -# # we can't load micro migrations on production -# end +ActiveRecord::Base.schema_format = :sql + +begin + ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + require 'micro_migrations' +rescue LoadError + # we can't load micro migrations on production +end + begin require 'rspec/core/rake_task' From 5353ed027e48d8e7d1908dacf55477fb603d4a07 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:12:15 +0100 Subject: [PATCH 04/45] add require travis --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index 016d761c..2eb5bee3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,5 @@ require 'bundler/setup' +require 'travis' require 'travis/migrations' require 'travis/engine' From 5e94e7436baa2def899b450255445d9ac812f624 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:23:37 +0100 Subject: [PATCH 05/45] omit structure dump unless dev env --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 2eb5bee3..b7e625dc 100644 --- a/Rakefile +++ b/Rakefile @@ -4,6 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql +Rake::Task["db:structure:dump"].clear unless Rails.env.development? begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -21,6 +22,7 @@ rescue LoadError warn "could not load rspec" end + desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From 8d98f39e792e86296a4f4c2b09b9d74024eae4ab Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:27:35 +0100 Subject: [PATCH 06/45] fix struture dump syntax --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index b7e625dc..3cd431ac 100644 --- a/Rakefile +++ b/Rakefile @@ -4,7 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear unless Rails.env.development? +Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) From 6b230828e68ae77d7503930edfffe958e5e5001d Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:32:40 +0100 Subject: [PATCH 07/45] move require micro-migrations to top of file --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 3cd431ac..be8fc7c7 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,6 @@ require 'bundler/setup' require 'travis' +require 'micro_migrations' require 'travis/migrations' require 'travis/engine' @@ -8,7 +9,6 @@ Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) - require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end From 66e63897a338de1a9a0886ad7d0200e44b1a9765 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:45:27 +0100 Subject: [PATCH 08/45] move structure dump --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index be8fc7c7..f97f5e20 100644 --- a/Rakefile +++ b/Rakefile @@ -5,7 +5,6 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -22,6 +21,7 @@ rescue LoadError warn "could not load rspec" end +Rake::Task["db:structure:dump"].clear desc "generate gemspec" task 'travis-api.gemspec' do From 6e8606c01ee673b0eefa835b4bb640a027524afc Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 20:53:04 +0100 Subject: [PATCH 09/45] restore to previous --- Rakefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Rakefile b/Rakefile index f97f5e20..9a143017 100644 --- a/Rakefile +++ b/Rakefile @@ -1,18 +1,14 @@ require 'bundler/setup' require 'travis' -require 'micro_migrations' -require 'travis/migrations' require 'travis/engine' -ActiveRecord::Base.schema_format = :sql - begin - ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end - begin require 'rspec/core/rake_task' RSpec::Core::RakeTask.new @@ -21,8 +17,6 @@ rescue LoadError warn "could not load rspec" end -Rake::Task["db:structure:dump"].clear - desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From b82ca5e438ce27b4a94af496e280a2bb1a5bebfa Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:10:21 +0100 Subject: [PATCH 10/45] update before script --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 08c1c58f..faa2b435 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,9 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - RAILS_ENV=test + - createdb travis_test + - bundle exec rake db:migrate --trace + +script: + - bundle exec rspec spec From 6f160912078b79a6b4ebc2cebf91bab824a69b61 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:13:12 +0100 Subject: [PATCH 11/45] update before script --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index faa2b435..c0470291 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ services: before_script: - RAILS_ENV=test - createdb travis_test + - createdb travis_pro_test - bundle exec rake db:migrate --trace script: From 172156da59c6c2c5f08fbe0374707319a93b1247 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:17:12 +0100 Subject: [PATCH 12/45] remove schema --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 9a143017..c4af9f66 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'travis' require 'travis/engine' begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) require 'micro_migrations' rescue LoadError # we can't load micro migrations on production From c0853547063f8bf1adc9be006849722b7242bae3 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:27:20 +0100 Subject: [PATCH 13/45] new rakefile --- .travis.yml | 5 +- Rakefile | 132 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 94 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0470291..58812bc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,7 @@ services: - redis before_script: - - RAILS_ENV=test - - createdb travis_test - - createdb travis_pro_test - - bundle exec rake db:migrate --trace + - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' script: - bundle exec rspec spec diff --git a/Rakefile b/Rakefile index c4af9f66..095cccec 100644 --- a/Rakefile +++ b/Rakefile @@ -1,47 +1,101 @@ +require 'rspec/core/rake_task' require 'bundler/setup' +require 'micro_migrations' require 'travis' -require 'travis/engine' -begin - # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production +ActiveRecord::Base.schema_format = :sql +Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] + +desc 'Run specs' +RSpec::Core::RakeTask.new do |t| + t.pattern = './spec/**/*_spec.rb' end -begin - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new - task default: :spec -rescue LoadError - warn "could not load rspec" -end - -desc "generate gemspec" -task 'travis-api.gemspec' do - content = File.read 'travis-api.gemspec' - - fields = { - authors: `git shortlog -sn`.scan(/[^\d\s].*/), - email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), - files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } - } - - fields.each do |field, values| - updated = " s.#{field} = [" - updated << values.map { |v| "\n %p" % v }.join(',') - updated << "\n ]" - content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) - end - - File.open('travis-api.gemspec', 'w') { |f| f << content } -end - -task default: 'travis-api.gemspec' - -tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) -Dir.glob(tasks_path).each { |r| import r } - +module ActiveRecord + class Migration + class << self + attr_accessor :disable_ddl_transaction + end + + # Disable DDL transactions for this migration. + def self.disable_ddl_transaction! + @disable_ddl_transaction = true + end + + def disable_ddl_transaction # :nodoc: + self.class.disable_ddl_transaction + end + end + + class Migrator + def use_transaction?(migration) + !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? + end + + def ddl_transaction(migration, &block) + if use_transaction?(migration) + Base.transaction { block.call } + else + block.call + end + end + + def migrate(&block) + current = migrations.detect { |m| m.version == current_version } + target = migrations.detect { |m| m.version == @target_version } + + if target.nil? && @target_version && @target_version > 0 + raise UnknownMigrationVersionError.new(@target_version) + end + + start = up? ? 0 : (migrations.index(current) || 0) + finish = migrations.index(target) || migrations.size - 1 + runnable = migrations[start..finish] + + # skip the last migration if we're headed down, but not ALL the way down + runnable.pop if down? && target + + ran = [] + runnable.each do |migration| + if block && !block.call(migration) + next + end + + Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger + + seen = migrated.include?(migration.version.to_i) + + # On our way up, we skip migrating the ones we've already migrated + next if up? && seen + + # On our way down, we skip reverting the ones we've never migrated + if down? && !seen + migration.announce 'never migrated, skipping'; migration.write + next + end + + begin + ddl_transaction(migration) do + migration.migrate(@direction) + record_version_state_after_migrating(migration.version) + end + ran << migration + rescue => e + canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" + raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace + end + end + ran + end + end + + class MigrationProxy + delegate :disable_ddl_transaction, to: :migration + end +end + +task :default => :spec + module ActiveRecord class Migration class << self From cafb078c10e3210293b2445a36f11a133026b214 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:36:32 +0100 Subject: [PATCH 14/45] add structure dump override --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 095cccec..f087e423 100644 --- a/Rakefile +++ b/Rakefile @@ -6,6 +6,8 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] + desc 'Run specs' RSpec::Core::RakeTask.new do |t| t.pattern = './spec/**/*_spec.rb' From 979201794f28845e7cbdf45fe8e6b2f915420654 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:41:40 +0100 Subject: [PATCH 15/45] fix dump override --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index f087e423..624d3300 100644 --- a/Rakefile +++ b/Rakefile @@ -6,7 +6,7 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] -Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] +Rake::Task["db:structure:dump"].clear unless Rails.env.development? desc 'Run specs' RSpec::Core::RakeTask.new do |t| From b16ae846ad409c5b00c2618a246d4d131c993338 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:49:17 +0100 Subject: [PATCH 16/45] try new rakefile --- Rakefile | 188 +++---------------------------------------------------- 1 file changed, 10 insertions(+), 178 deletions(-) diff --git a/Rakefile b/Rakefile index 624d3300..e6eecf07 100644 --- a/Rakefile +++ b/Rakefile @@ -1,182 +1,14 @@ -require 'rspec/core/rake_task' -require 'bundler/setup' -require 'micro_migrations' -require 'travis' +require 'rake' +require 'travis/migrations' -ActiveRecord::Base.schema_format = :sql -Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +task default: :spec -Rake::Task["db:structure:dump"].clear unless Rails.env.development? - -desc 'Run specs' -RSpec::Core::RakeTask.new do |t| - t.pattern = './spec/**/*_spec.rb' -end - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration - end -end - -task :default => :spec - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration +namespace :db do + desc 'Create the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh 'mkdir spec/support/db' + sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" + sh 'psql -q travis_test < spec/support/db/structure.sql' end end From 54e2ca0300130e06ff0bdd13c06f64dd0f4a5e3c Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:52:02 +0100 Subject: [PATCH 17/45] remove db:migrate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58812bc5..3f371b9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - 'RAILS_ENV=test bundle exec rake db:create --trace' script: - bundle exec rspec spec From 8d1cd15ee23a39546d2d2e6c5a3629c42707a88b Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:59:35 +0100 Subject: [PATCH 18/45] restore original db config --- config/database.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/database.yml b/config/database.yml index 53e59a34..97ba5e01 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,8 +15,8 @@ production: development: <<: *defaults - database: travis_pro_development + database: travis_development test: <<: *defaults - database: travis_pro_test + database: travis_test From 079cec3467eda24d937e8ebdc4f236e5dd9c0a27 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 5 Feb 2016 02:51:30 +0100 Subject: [PATCH 19/45] reinstate rake tasks that are still needed --- Rakefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Rakefile b/Rakefile index e6eecf07..224ae1fb 100644 --- a/Rakefile +++ b/Rakefile @@ -12,3 +12,28 @@ namespace :db do sh 'psql -q travis_test < spec/support/db/structure.sql' end end + +desc "generate gemspec" +task 'travis-api.gemspec' do + content = File.read 'travis-api.gemspec' + + fields = { + authors: `git shortlog -sn`.scan(/[^\d\s].*/), + email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), + files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } + } + + fields.each do |field, values| + updated = " s.#{field} = [" + updated << values.map { |v| "\n %p" % v }.join(',') + updated << "\n ]" + content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) + end + + File.open('travis-api.gemspec', 'w') { |f| f << content } + end + + task default: 'travis-api.gemspec' + + tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) + Dir.glob(tasks_path).each { |r| import r } From 277562e51c979c66b4e63e039549bb55d65c09c0 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 11 Feb 2016 10:13:05 +0100 Subject: [PATCH 20/45] add db creation and migration for development, update readme --- README.md | 8 ++++---- Rakefile | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b72bb7c8..9b02729f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the app running on https://api.travis-ci.org/ 1. PostgreSQL 9.3 or higher 1. Redis 1. RabbitMQ -1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. +1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. ## Installation @@ -17,8 +17,8 @@ This is the app running on https://api.travis-ci.org/ ### Database setup -1. `rake db:create db:migrate` -2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' +1. `bundle exec rake db:create` +2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): ```sh-session cd .. @@ -31,7 +31,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development Repeat the database steps for `RAILS_ENV=test`. ```sh-session -RAILS_ENV=test rake db:create db:structure:load +RAILS_ENV=test bundle exec rake db:create pushd ../travis-logs RAILS_ENV=test rvm jruby do bundle exec rake db:migrate psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test diff --git a/Rakefile b/Rakefile index 224ae1fb..93c7d171 100644 --- a/Rakefile +++ b/Rakefile @@ -4,12 +4,18 @@ require 'travis/migrations' task default: :spec namespace :db do - desc 'Create the test database' - task :create do - sh 'createdb travis_test' rescue nil - sh 'mkdir spec/support/db' - sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" - sh 'psql -q travis_test < spec/support/db/structure.sql' + if ENV["RAILS_ENV"] == 'test' + desc 'Create and migrate the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end + else + desc 'Create and migrate the development database' + task :create do + sh 'createdb travis_development' rescue nil + sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end end end From 693e2939092fc1ce0b1a56a1aac2a7a19772c6ad Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Wed, 16 Dec 2015 14:31:04 +0100 Subject: [PATCH 21/45] make sure params is always a hash, fixes exceptions --- Gemfile.lock | 3 --- lib/travis/api/app/helpers/accept.rb | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index dbc31374..f4f974f9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -377,6 +377,3 @@ DEPENDENCIES travis-yaml! unicorn yard-sinatra! - -BUNDLED WITH - 1.10.6 diff --git a/lib/travis/api/app/helpers/accept.rb b/lib/travis/api/app/helpers/accept.rb index 5fb41c67..56d7fd86 100644 --- a/lib/travis/api/app/helpers/accept.rb +++ b/lib/travis/api/app/helpers/accept.rb @@ -53,6 +53,8 @@ class Travis::Api::App if params params = Hash[*params.split(';').map { |p| p.scan /(#{TOKEN})=(#{TOKEN})/ }.flatten] quality = params.delete('q').to_f if params['q'] + else + params = {} end if subtype =~ HEADER_FORMAT From 8c3c4fe935bd07e63fbcf856a25abb29a82ed97d Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Wed, 16 Dec 2015 15:06:15 +0100 Subject: [PATCH 22/45] add ip whitelisting --- lib/travis/api/attack.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index dc0a247d..207396f5 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -31,6 +31,12 @@ class Rack::Attack "/auth/post_message/iframe" ] + #### + # Whitelisted IP addresses + whitelist('whitelist client requesting from redis') do |request| + Travis.redis.sismember(:api_whitelisted_ips, request.ip) + end + #### # Ban based on: IP address # Ban time: indefinite From 416fcf2d4b8adefa992785b06a6eff00fbc832b3 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Wed, 16 Dec 2015 15:10:03 +0100 Subject: [PATCH 23/45] Ruby 2.1.6 is no longer supported on Heroku --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index ce0fe1b8..e50f9b90 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org' gemspec -ruby '2.1.6' if ENV.key?('DYNO') +ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' From a66191c6829bec31e982ee99cae47e91211f65cf Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 15 Jan 2016 15:45:26 +0100 Subject: [PATCH 24/45] bump travis core --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f4f974f9..8c981f56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 3a9f6e4c14bb1eacb93609e1864c9c547a13c1a4 + revision: 96ee8c449ebe305c5c95633cea13eb88fe978abb specs: travis-core (0.0.1) actionmailer (~> 3.2.19) From 797ca9f9fac2514a937cf6b58b8770c61e8bfea9 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 22 Jan 2016 14:26:30 +0100 Subject: [PATCH 25/45] v3: add metrics --- Gemfile | 1 + Gemfile.lock | 4 ++ lib/travis/api/app.rb | 4 +- lib/travis/api/v3/metrics.rb | 103 +++++++++++++++++++++++++++++++++++ lib/travis/api/v3/router.rb | 35 +++++++++--- spec/spec_helper.rb | 1 + spec/v3/metrics_spec.rb | 42 ++++++++++++++ 7 files changed, 181 insertions(+), 9 deletions(-) create mode 100644 lib/travis/api/v3/metrics.rb create mode 100644 spec/v3/metrics_spec.rb diff --git a/Gemfile b/Gemfile index e50f9b90..9893ebf5 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,7 @@ gem 'customerio' group :test do gem 'rspec', '~> 2.13' + gem 'rspec-its' gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8c981f56..09ee17b3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -276,6 +276,9 @@ GEM rspec-core (2.99.2) rspec-expectations (2.99.2) diff-lcs (>= 1.1.3, < 2.0) + rspec-its (1.0.1) + rspec-core (>= 2.99.0.beta1) + rspec-expectations (>= 2.99.0.beta1) rspec-mocks (2.99.2) sidekiq (3.3.0) celluloid (>= 0.16.0) @@ -362,6 +365,7 @@ DEPENDENCIES rb-fsevent (~> 0.9.1) rerun rspec (~> 2.13) + rspec-its s3! sentry-raven! simplecov diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index a2763fdd..edc64310 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -122,7 +122,6 @@ module Travis::Api use Travis::Api::App::Middleware::Logging use Travis::Api::App::Middleware::ScopeCheck use Travis::Api::App::Middleware::UserAgentTracker - use Travis::Api::App::Middleware::Metriks # make sure this is below ScopeCheck so we have the token use Rack::Attack if Endpoint.production? @@ -133,6 +132,9 @@ module Travis::Api # rewrite should come after V3 hook use Travis::Api::App::Middleware::Rewrite + # v3 has its own metriks + use Travis::Api::App::Middleware::Metriks + SettingsEndpoint.subclass :env_vars if Travis.config.endpoints.ssh_key SingletonSettingsEndpoint.subclass :ssh_key diff --git a/lib/travis/api/v3/metrics.rb b/lib/travis/api/v3/metrics.rb new file mode 100644 index 00000000..6bf1bd3a --- /dev/null +++ b/lib/travis/api/v3/metrics.rb @@ -0,0 +1,103 @@ +require 'metriks' + +module Travis::API::V3 + class Metrics + class MetriksTracker + def initialize(prefix: "api.v3") + @prefx = prefix + end + + def time(name, duration) + ::Metriks.timer("#{@prefix}.#{name}").update(duration) + end + + def mark(name) + ::Metriks.meter("#{@prefix}.#{name}").mark + end + end + + class Processor + attr_reader :queue, :tracker + + def initialize(queue_size: 1000, tracker: MetriksTracker.new) + @tracker = tracker + @queue = queue_size ? ::SizedQueue.new(queue_size) : ::Queue.new + end + + def create(**options) + Metrics.new(self, **options) + end + + def start + Thread.new { loop { process(queue.pop) } } + end + + def process(metrics) + metrics.process(tracker) + rescue Exception => e + $stderr.puts e.message, e.backtrace + end + end + + def initialize(processor, time: Time.now) + @processor = processor + @start_time = time + @name_after = nil + @ticks = [] + @success = nil + @name = "unknown".freeze + end + + def tick(event, time: Time.now) + @ticks << [event, time] + self + end + + def success(**options) + finish(true, **options) + end + + def failure(**options) + finish(false, **options) + end + + def name_after(factory) + @name = nil + @name_after = factory + self + end + + def finish(success, time: Time.now, status: nil) + @success = !!success + @status = status + @status ||= success ? 200 : 500 + @end_time = time + @processor.queue << self + self + end + + def name + @name ||= @name_after.name[/[^:]+::[^:]+$/].underscore.tr(?/.freeze, ?..freeze) + end + + def process(tracker) + tracker.mark("status.#{@status}") + + if @success + process_ticks(tracker) + tracker.time("#{name}.overall", @end_time - @start_time) + tracker.mark("#{name}.success") + else + tracker.mark("#{name}.failure") + end + end + + def process_ticks(tracker) + start = @start_time + @ticks.each do |event, time| + tracker.time("#{name}.#{event}", time - start) + start = time + end + end + end +end diff --git a/lib/travis/api/v3/router.rb b/lib/travis/api/v3/router.rb index 8e39fc15..573c7c85 100644 --- a/lib/travis/api/v3/router.rb +++ b/lib/travis/api/v3/router.rb @@ -1,30 +1,49 @@ module Travis::API::V3 class Router include Travis::API::V3 - attr_accessor :routes + attr_accessor :routes, :metrics_processor def initialize(routes = Routes) - @routes = routes + @routes = routes + @metrics_processor = Metrics::Processor.new + + metrics_processor.start routes.draw_routes end def call(env) return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze + metrics = @metrics_processor.create access_control = AccessControl.new(env) factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) env_params = params(env) raise NotFound unless factory + metrics.name_after(factory) - filtered = factory.filter_params(env_params) - service = factory.new(access_control, filtered.merge(params)) - result = service.run + filtered = factory.filter_params(env_params) + service = factory.new(access_control, filtered.merge(params)) + + metrics.tick(:prepare) + result = service.run + metrics.tick(:service) env_params.each_key { |key| result.ignored_param(key, reason: "not whitelisted".freeze) unless filtered.include?(key) } - render(result, env_params, env) + response = render(result, env_params, env) + + metrics.tick(:renderer) + metrics.success(status: response[0]) + response rescue Error => error - result = Result.new(access_control, :error, error) - V3.response(result.render(env_params, env), {}, status: error.status) + metrics.tick(:service) + + result = Result.new(access_control, :error, error) + response = V3.response(result.render(env_params, env), {}, status: error.status) + + metrics.tick(:rendered) + metrics.failure(status: error.status) + + response end def render(result, env_params, env) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ced5e7af..5161936e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -3,6 +3,7 @@ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = ENV['ENV'] = 'test' require 'support/coverage' require 'rspec' +require 'rspec/its' require 'database_cleaner' require 'sinatra/test_helpers' require 'logger' diff --git a/spec/v3/metrics_spec.rb b/spec/v3/metrics_spec.rb new file mode 100644 index 00000000..a5afe054 --- /dev/null +++ b/spec/v3/metrics_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe Travis::API::V3::Metrics do + class TestProcessor + attr_reader :times, :marks, :queue + + def initialize + @queue = [] + @times = [] + @marks = [] + end + + def time(*args) + times << args + end + + def mark(name) + marks << name + end + end + + subject(:processor) { TestProcessor.new } + let(:metric) { described_class.new(processor, time: Time.at(0)) } + + before do + metric.name_after(Travis::API::V3::Services::Branch::Find) + metric.tick(:example, time: Time.at(10)) + metric.tick(:other_example, time: Time.at(15)) + metric.success(time: Time.at(25)) + metric.process(processor) + end + + its(:queue) { should be == [metric] } + + its(:times) { should be == [ + ["branch.find.example", 10.0], + ["branch.find.other_example", 5.0], + ["branch.find.overall", 25.0] + ] } + + its(:marks) { should be == ["status.200", "branch.find.success"] } +end From 5f32123891c0417ad38fedf01851e2eb3aad9551 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:03:09 +0100 Subject: [PATCH 26/45] comment out schema env definition --- Rakefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Rakefile b/Rakefile index 9a143017..58566cbf 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,13 @@ require 'bundler/setup' -require 'travis' +# require 'travis' require 'travis/engine' -begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production -end +# begin +# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) +# require 'micro_migrations' +# rescue LoadError +# # we can't load micro migrations on production +# end begin require 'rspec/core/rake_task' From 6c5c48316634570b91fe8b799a1dfb3c07b873b3 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:08:31 +0100 Subject: [PATCH 27/45] use sql schema --- Gemfile | 1 + Gemfile.lock | 10 ++++++++++ Rakefile | 17 ++++++++++------- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 9893ebf5..bf2e564b 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ group :test do gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' + gem 'travis-migrations', github: 'travis-ci/travis-migrations' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 09ee17b3..a340c150 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -72,6 +72,12 @@ GIT travis-config (~> 0.1.0) virtus (~> 1.0.0) +GIT + remote: git://github.com/travis-ci/travis-migrations.git + revision: d6105ad77303bcd04d1e376070a11dbc8b3077c7 + specs: + travis-migrations (0.0.1) + GIT remote: git://github.com/travis-ci/travis-sidekiqs.git revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23 @@ -376,8 +382,12 @@ DEPENDENCIES travis-api! travis-config (~> 0.1.0) travis-core! + travis-migrations! travis-sidekiqs! travis-support! travis-yaml! unicorn yard-sinatra! + +BUNDLED WITH + 1.10.6 diff --git a/Rakefile b/Rakefile index 58566cbf..016d761c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,16 @@ require 'bundler/setup' -# require 'travis' +require 'travis/migrations' require 'travis/engine' -# begin -# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) -# require 'micro_migrations' -# rescue LoadError -# # we can't load micro migrations on production -# end +ActiveRecord::Base.schema_format = :sql + +begin + ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + require 'micro_migrations' +rescue LoadError + # we can't load micro migrations on production +end + begin require 'rspec/core/rake_task' From c13cabfbf3d4052d8a7d6377156cf8123e9eb611 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:12:15 +0100 Subject: [PATCH 28/45] add require travis --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index 016d761c..2eb5bee3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,5 @@ require 'bundler/setup' +require 'travis' require 'travis/migrations' require 'travis/engine' From 16db16269634bd5675eca90f0921ade855ae6eff Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:23:37 +0100 Subject: [PATCH 29/45] omit structure dump unless dev env --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 2eb5bee3..b7e625dc 100644 --- a/Rakefile +++ b/Rakefile @@ -4,6 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql +Rake::Task["db:structure:dump"].clear unless Rails.env.development? begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -21,6 +22,7 @@ rescue LoadError warn "could not load rspec" end + desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From 84cfacadb6177e000bf87f0b38a96f20b0276192 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:27:35 +0100 Subject: [PATCH 30/45] fix struture dump syntax --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index b7e625dc..3cd431ac 100644 --- a/Rakefile +++ b/Rakefile @@ -4,7 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear unless Rails.env.development? +Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) From 15e3b84dca1e7f11b68d5c276a53a355852502f5 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:32:40 +0100 Subject: [PATCH 31/45] move require micro-migrations to top of file --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 3cd431ac..be8fc7c7 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,6 @@ require 'bundler/setup' require 'travis' +require 'micro_migrations' require 'travis/migrations' require 'travis/engine' @@ -8,7 +9,6 @@ Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) - require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end From 345921a98b74a1ea1046dba60632a6c20d0b0d8d Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:45:27 +0100 Subject: [PATCH 32/45] move structure dump --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index be8fc7c7..f97f5e20 100644 --- a/Rakefile +++ b/Rakefile @@ -5,7 +5,6 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -22,6 +21,7 @@ rescue LoadError warn "could not load rspec" end +Rake::Task["db:structure:dump"].clear desc "generate gemspec" task 'travis-api.gemspec' do From 1fb526de7efc8a9b4ace8071b9d0b88950ae840f Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 20:53:04 +0100 Subject: [PATCH 33/45] restore to previous --- Rakefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Rakefile b/Rakefile index f97f5e20..9a143017 100644 --- a/Rakefile +++ b/Rakefile @@ -1,18 +1,14 @@ require 'bundler/setup' require 'travis' -require 'micro_migrations' -require 'travis/migrations' require 'travis/engine' -ActiveRecord::Base.schema_format = :sql - begin - ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end - begin require 'rspec/core/rake_task' RSpec::Core::RakeTask.new @@ -21,8 +17,6 @@ rescue LoadError warn "could not load rspec" end -Rake::Task["db:structure:dump"].clear - desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From 179b166de4757e6dfa88c8b7c5a02e52a6278eb8 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:10:21 +0100 Subject: [PATCH 34/45] update before script --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 08c1c58f..faa2b435 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,9 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - RAILS_ENV=test + - createdb travis_test + - bundle exec rake db:migrate --trace + +script: + - bundle exec rspec spec From e2e222ab7cf2bfde7172d41673d331d7920f8fdb Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:13:12 +0100 Subject: [PATCH 35/45] update before script --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index faa2b435..c0470291 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ services: before_script: - RAILS_ENV=test - createdb travis_test + - createdb travis_pro_test - bundle exec rake db:migrate --trace script: From 14ec02da79e183cbc38e80aaa58939198d38bddf Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:17:12 +0100 Subject: [PATCH 36/45] remove schema --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 9a143017..c4af9f66 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'travis' require 'travis/engine' begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) require 'micro_migrations' rescue LoadError # we can't load micro migrations on production From 4d98ccafd64b33794294c4043ee41a19d72881b2 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:27:20 +0100 Subject: [PATCH 37/45] new rakefile --- .travis.yml | 5 +- Rakefile | 132 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 94 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0470291..58812bc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,7 @@ services: - redis before_script: - - RAILS_ENV=test - - createdb travis_test - - createdb travis_pro_test - - bundle exec rake db:migrate --trace + - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' script: - bundle exec rspec spec diff --git a/Rakefile b/Rakefile index c4af9f66..095cccec 100644 --- a/Rakefile +++ b/Rakefile @@ -1,47 +1,101 @@ +require 'rspec/core/rake_task' require 'bundler/setup' +require 'micro_migrations' require 'travis' -require 'travis/engine' -begin - # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production +ActiveRecord::Base.schema_format = :sql +Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] + +desc 'Run specs' +RSpec::Core::RakeTask.new do |t| + t.pattern = './spec/**/*_spec.rb' end -begin - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new - task default: :spec -rescue LoadError - warn "could not load rspec" -end - -desc "generate gemspec" -task 'travis-api.gemspec' do - content = File.read 'travis-api.gemspec' - - fields = { - authors: `git shortlog -sn`.scan(/[^\d\s].*/), - email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), - files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } - } - - fields.each do |field, values| - updated = " s.#{field} = [" - updated << values.map { |v| "\n %p" % v }.join(',') - updated << "\n ]" - content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) - end - - File.open('travis-api.gemspec', 'w') { |f| f << content } -end - -task default: 'travis-api.gemspec' - -tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) -Dir.glob(tasks_path).each { |r| import r } - +module ActiveRecord + class Migration + class << self + attr_accessor :disable_ddl_transaction + end + + # Disable DDL transactions for this migration. + def self.disable_ddl_transaction! + @disable_ddl_transaction = true + end + + def disable_ddl_transaction # :nodoc: + self.class.disable_ddl_transaction + end + end + + class Migrator + def use_transaction?(migration) + !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? + end + + def ddl_transaction(migration, &block) + if use_transaction?(migration) + Base.transaction { block.call } + else + block.call + end + end + + def migrate(&block) + current = migrations.detect { |m| m.version == current_version } + target = migrations.detect { |m| m.version == @target_version } + + if target.nil? && @target_version && @target_version > 0 + raise UnknownMigrationVersionError.new(@target_version) + end + + start = up? ? 0 : (migrations.index(current) || 0) + finish = migrations.index(target) || migrations.size - 1 + runnable = migrations[start..finish] + + # skip the last migration if we're headed down, but not ALL the way down + runnable.pop if down? && target + + ran = [] + runnable.each do |migration| + if block && !block.call(migration) + next + end + + Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger + + seen = migrated.include?(migration.version.to_i) + + # On our way up, we skip migrating the ones we've already migrated + next if up? && seen + + # On our way down, we skip reverting the ones we've never migrated + if down? && !seen + migration.announce 'never migrated, skipping'; migration.write + next + end + + begin + ddl_transaction(migration) do + migration.migrate(@direction) + record_version_state_after_migrating(migration.version) + end + ran << migration + rescue => e + canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" + raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace + end + end + ran + end + end + + class MigrationProxy + delegate :disable_ddl_transaction, to: :migration + end +end + +task :default => :spec + module ActiveRecord class Migration class << self From 25bd6b88b743dfaa080e29d5aae38bb238e04cfe Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:36:32 +0100 Subject: [PATCH 38/45] add structure dump override --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 095cccec..f087e423 100644 --- a/Rakefile +++ b/Rakefile @@ -6,6 +6,8 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] + desc 'Run specs' RSpec::Core::RakeTask.new do |t| t.pattern = './spec/**/*_spec.rb' From e283140d02ebca39ec88abff47c65de0371bdd5b Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:41:40 +0100 Subject: [PATCH 39/45] fix dump override --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index f087e423..624d3300 100644 --- a/Rakefile +++ b/Rakefile @@ -6,7 +6,7 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] -Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] +Rake::Task["db:structure:dump"].clear unless Rails.env.development? desc 'Run specs' RSpec::Core::RakeTask.new do |t| From c1d6441c7eb52a660659b375120488e21e203268 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:49:17 +0100 Subject: [PATCH 40/45] try new rakefile --- Rakefile | 188 +++---------------------------------------------------- 1 file changed, 10 insertions(+), 178 deletions(-) diff --git a/Rakefile b/Rakefile index 624d3300..e6eecf07 100644 --- a/Rakefile +++ b/Rakefile @@ -1,182 +1,14 @@ -require 'rspec/core/rake_task' -require 'bundler/setup' -require 'micro_migrations' -require 'travis' +require 'rake' +require 'travis/migrations' -ActiveRecord::Base.schema_format = :sql -Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +task default: :spec -Rake::Task["db:structure:dump"].clear unless Rails.env.development? - -desc 'Run specs' -RSpec::Core::RakeTask.new do |t| - t.pattern = './spec/**/*_spec.rb' -end - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration - end -end - -task :default => :spec - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration +namespace :db do + desc 'Create the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh 'mkdir spec/support/db' + sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" + sh 'psql -q travis_test < spec/support/db/structure.sql' end end From 14b7159109d13e224c5928126c5103c0df1ed508 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:52:02 +0100 Subject: [PATCH 41/45] remove db:migrate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58812bc5..3f371b9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - 'RAILS_ENV=test bundle exec rake db:create --trace' script: - bundle exec rspec spec From 9744f59fe276d9847beeeeb24f421e2f4432d2ec Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:59:35 +0100 Subject: [PATCH 42/45] restore original db config --- config/database.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/database.yml b/config/database.yml index 53e59a34..97ba5e01 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,8 +15,8 @@ production: development: <<: *defaults - database: travis_pro_development + database: travis_development test: <<: *defaults - database: travis_pro_test + database: travis_test From 6ab0ec3c3a33103b7ef662eb5627a35f8fdb3d88 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 5 Feb 2016 02:51:30 +0100 Subject: [PATCH 43/45] reinstate rake tasks that are still needed --- Rakefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Rakefile b/Rakefile index e6eecf07..224ae1fb 100644 --- a/Rakefile +++ b/Rakefile @@ -12,3 +12,28 @@ namespace :db do sh 'psql -q travis_test < spec/support/db/structure.sql' end end + +desc "generate gemspec" +task 'travis-api.gemspec' do + content = File.read 'travis-api.gemspec' + + fields = { + authors: `git shortlog -sn`.scan(/[^\d\s].*/), + email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), + files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } + } + + fields.each do |field, values| + updated = " s.#{field} = [" + updated << values.map { |v| "\n %p" % v }.join(',') + updated << "\n ]" + content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) + end + + File.open('travis-api.gemspec', 'w') { |f| f << content } + end + + task default: 'travis-api.gemspec' + + tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) + Dir.glob(tasks_path).each { |r| import r } From 64f97a89d24a589828b0dbf58a417290951fb6e4 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 11 Feb 2016 10:13:05 +0100 Subject: [PATCH 44/45] add db creation and migration for development, update readme --- README.md | 8 ++++---- Rakefile | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b72bb7c8..9b02729f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the app running on https://api.travis-ci.org/ 1. PostgreSQL 9.3 or higher 1. Redis 1. RabbitMQ -1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. +1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. ## Installation @@ -17,8 +17,8 @@ This is the app running on https://api.travis-ci.org/ ### Database setup -1. `rake db:create db:migrate` -2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' +1. `bundle exec rake db:create` +2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): ```sh-session cd .. @@ -31,7 +31,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development Repeat the database steps for `RAILS_ENV=test`. ```sh-session -RAILS_ENV=test rake db:create db:structure:load +RAILS_ENV=test bundle exec rake db:create pushd ../travis-logs RAILS_ENV=test rvm jruby do bundle exec rake db:migrate psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test diff --git a/Rakefile b/Rakefile index 224ae1fb..93c7d171 100644 --- a/Rakefile +++ b/Rakefile @@ -4,12 +4,18 @@ require 'travis/migrations' task default: :spec namespace :db do - desc 'Create the test database' - task :create do - sh 'createdb travis_test' rescue nil - sh 'mkdir spec/support/db' - sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" - sh 'psql -q travis_test < spec/support/db/structure.sql' + if ENV["RAILS_ENV"] == 'test' + desc 'Create and migrate the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end + else + desc 'Create and migrate the development database' + task :create do + sh 'createdb travis_development' rescue nil + sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end end end From 54572a5dede033c9f659fcd56505accf48ce532d Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 24 Feb 2016 23:42:50 +0100 Subject: [PATCH 45/45] update readme with info about db name --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9b02729f..5f6aaee5 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ This is the app running on https://api.travis-ci.org/ ### Database setup +NB detail for how `rake` sets up the database can be found in the `Rakefile`. In the `namespace :db` block you will see the database name for development is hardcoded to `travis-development`. If you are using a different configuration you will have to make your own adjustments. + 1. `bundle exec rake db:create` 2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`):