From 7eb2617e48449080f1f50d754b5268dd8a440ca6 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Tue, 24 Nov 2015 15:50:38 +0100 Subject: [PATCH 01/88] Add branch_is_default info to commits --- lib/travis/api/v2/http/build.rb | 9 +++++++-- lib/travis/api/v2/http/job.rb | 9 +++++++-- spec/unit/api/v2/http/build_spec.rb | 1 + spec/unit/api/v2/http/job_spec.rb | 1 + 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/travis/api/v2/http/build.rb b/lib/travis/api/v2/http/build.rb index 9d8203b5..394c4ac8 100644 --- a/lib/travis/api/v2/http/build.rb +++ b/lib/travis/api/v2/http/build.rb @@ -17,7 +17,7 @@ module Travis def data { 'build' => build_data(build), - 'commit' => commit_data(build.commit), + 'commit' => commit_data(build.commit, build.repository), 'jobs' => options[:include_jobs] ? build.matrix.map { |job| job_data(job) } : [], 'annotations' => options[:include_jobs] ? Annotations.new(annotations(build), @options).data["annotations"] : [], } @@ -44,11 +44,12 @@ module Travis } end - def commit_data(commit) + def commit_data(commit, repository) { 'id' => commit.id, 'sha' => commit.commit, 'branch' => commit.branch, + 'branch_is_default' => branch_is_default(commit, repository), 'message' => commit.message, 'committed_at' => format_date(commit.committed_at), 'author_name' => commit.author_name, @@ -78,6 +79,10 @@ module Travis } end + def branch_is_default(commit, repository) + repository.default_branch == commit.branch + end + def annotations(build) build.matrix.map(&:annotations).flatten end diff --git a/lib/travis/api/v2/http/job.rb b/lib/travis/api/v2/http/job.rb index 4b712b40..92366474 100644 --- a/lib/travis/api/v2/http/job.rb +++ b/lib/travis/api/v2/http/job.rb @@ -15,7 +15,7 @@ module Travis def data { 'job' => job_data(job), - 'commit' => commit_data(job.commit), + 'commit' => commit_data(job.commit, job.repository), 'annotations' => Annotations.new(job.annotations, @options).data["annotations"], } end @@ -42,11 +42,12 @@ module Travis } end - def commit_data(commit) + def commit_data(commit, repository) { 'id' => commit.id, 'sha' => commit.commit, 'branch' => commit.branch, + 'branch_is_default' => branch_is_default(commit, repository), 'message' => commit.message, 'committed_at' => format_date(commit.committed_at), 'author_name' => commit.author_name, @@ -56,6 +57,10 @@ module Travis 'compare_url' => commit.compare_url, } end + + def branch_is_default(commit, repository) + repository.default_branch == commit.branch + end end end end diff --git a/spec/unit/api/v2/http/build_spec.rb b/spec/unit/api/v2/http/build_spec.rb index 11a6dbea..f276b2b8 100644 --- a/spec/unit/api/v2/http/build_spec.rb +++ b/spec/unit/api/v2/http/build_spec.rb @@ -29,6 +29,7 @@ describe Travis::Api::V2::Http::Build do 'id' => 1, 'sha' => '62aae5f70ceee39123ef', 'branch' => 'master', + 'branch_is_default' => true, 'message' => 'the commit message', 'compare_url' => 'https://github.com/svenfuchs/minimal/compare/master...develop', 'committed_at' => json_format_time(Time.now.utc - 1.hour), diff --git a/spec/unit/api/v2/http/job_spec.rb b/spec/unit/api/v2/http/job_spec.rb index a9211b6f..22fda7b6 100644 --- a/spec/unit/api/v2/http/job_spec.rb +++ b/spec/unit/api/v2/http/job_spec.rb @@ -31,6 +31,7 @@ describe Travis::Api::V2::Http::Job do 'sha' => '62aae5f70ceee39123ef', 'message' => 'the commit message', 'branch' => 'master', + 'branch_is_default' => true, 'message' => 'the commit message', 'committed_at' => json_format_time(Time.now.utc - 1.hour), 'committer_name' => 'Sven Fuchs', From 759fdf203b0ee39e9d0e15d68ae904a96ace1b3f Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 11 Dec 2015 12:35:37 +0100 Subject: [PATCH 02/88] change rakefile and db congig --- Rakefile | 1 - config/database.yml | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Rakefile b/Rakefile index 866a2958..9a143017 100644 --- a/Rakefile +++ b/Rakefile @@ -8,7 +8,6 @@ begin rescue LoadError # we can't load micro migrations on production end -require 'travis' begin require 'rspec/core/rake_task' diff --git a/config/database.yml b/config/database.yml index 26b09327..53e59a34 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,9 +15,8 @@ production: development: <<: *defaults - database: travis_development + database: travis_pro_development test: <<: *defaults - database: travis_test - + database: travis_pro_test From bf1714c437b0c167f324ffebb81ce74b2c1ae0af Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 22 Jan 2016 14:26:30 +0100 Subject: [PATCH 03/88] v3: add metrics --- Gemfile | 1 + Gemfile.lock | 4 ++ lib/travis/api/app.rb | 4 +- lib/travis/api/v3/metrics.rb | 103 +++++++++++++++++++++++++++++++++++ lib/travis/api/v3/router.rb | 35 +++++++++--- spec/spec_helper.rb | 1 + spec/v3/metrics_spec.rb | 42 ++++++++++++++ 7 files changed, 181 insertions(+), 9 deletions(-) create mode 100644 lib/travis/api/v3/metrics.rb create mode 100644 spec/v3/metrics_spec.rb diff --git a/Gemfile b/Gemfile index e50f9b90..9893ebf5 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,7 @@ gem 'customerio' group :test do gem 'rspec', '~> 2.13' + gem 'rspec-its' gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8c981f56..09ee17b3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -276,6 +276,9 @@ GEM rspec-core (2.99.2) rspec-expectations (2.99.2) diff-lcs (>= 1.1.3, < 2.0) + rspec-its (1.0.1) + rspec-core (>= 2.99.0.beta1) + rspec-expectations (>= 2.99.0.beta1) rspec-mocks (2.99.2) sidekiq (3.3.0) celluloid (>= 0.16.0) @@ -362,6 +365,7 @@ DEPENDENCIES rb-fsevent (~> 0.9.1) rerun rspec (~> 2.13) + rspec-its s3! sentry-raven! simplecov diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index a2763fdd..edc64310 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -122,7 +122,6 @@ module Travis::Api use Travis::Api::App::Middleware::Logging use Travis::Api::App::Middleware::ScopeCheck use Travis::Api::App::Middleware::UserAgentTracker - use Travis::Api::App::Middleware::Metriks # make sure this is below ScopeCheck so we have the token use Rack::Attack if Endpoint.production? @@ -133,6 +132,9 @@ module Travis::Api # rewrite should come after V3 hook use Travis::Api::App::Middleware::Rewrite + # v3 has its own metriks + use Travis::Api::App::Middleware::Metriks + SettingsEndpoint.subclass :env_vars if Travis.config.endpoints.ssh_key SingletonSettingsEndpoint.subclass :ssh_key diff --git a/lib/travis/api/v3/metrics.rb b/lib/travis/api/v3/metrics.rb new file mode 100644 index 00000000..6bf1bd3a --- /dev/null +++ b/lib/travis/api/v3/metrics.rb @@ -0,0 +1,103 @@ +require 'metriks' + +module Travis::API::V3 + class Metrics + class MetriksTracker + def initialize(prefix: "api.v3") + @prefx = prefix + end + + def time(name, duration) + ::Metriks.timer("#{@prefix}.#{name}").update(duration) + end + + def mark(name) + ::Metriks.meter("#{@prefix}.#{name}").mark + end + end + + class Processor + attr_reader :queue, :tracker + + def initialize(queue_size: 1000, tracker: MetriksTracker.new) + @tracker = tracker + @queue = queue_size ? ::SizedQueue.new(queue_size) : ::Queue.new + end + + def create(**options) + Metrics.new(self, **options) + end + + def start + Thread.new { loop { process(queue.pop) } } + end + + def process(metrics) + metrics.process(tracker) + rescue Exception => e + $stderr.puts e.message, e.backtrace + end + end + + def initialize(processor, time: Time.now) + @processor = processor + @start_time = time + @name_after = nil + @ticks = [] + @success = nil + @name = "unknown".freeze + end + + def tick(event, time: Time.now) + @ticks << [event, time] + self + end + + def success(**options) + finish(true, **options) + end + + def failure(**options) + finish(false, **options) + end + + def name_after(factory) + @name = nil + @name_after = factory + self + end + + def finish(success, time: Time.now, status: nil) + @success = !!success + @status = status + @status ||= success ? 200 : 500 + @end_time = time + @processor.queue << self + self + end + + def name + @name ||= @name_after.name[/[^:]+::[^:]+$/].underscore.tr(?/.freeze, ?..freeze) + end + + def process(tracker) + tracker.mark("status.#{@status}") + + if @success + process_ticks(tracker) + tracker.time("#{name}.overall", @end_time - @start_time) + tracker.mark("#{name}.success") + else + tracker.mark("#{name}.failure") + end + end + + def process_ticks(tracker) + start = @start_time + @ticks.each do |event, time| + tracker.time("#{name}.#{event}", time - start) + start = time + end + end + end +end diff --git a/lib/travis/api/v3/router.rb b/lib/travis/api/v3/router.rb index 8e39fc15..573c7c85 100644 --- a/lib/travis/api/v3/router.rb +++ b/lib/travis/api/v3/router.rb @@ -1,30 +1,49 @@ module Travis::API::V3 class Router include Travis::API::V3 - attr_accessor :routes + attr_accessor :routes, :metrics_processor def initialize(routes = Routes) - @routes = routes + @routes = routes + @metrics_processor = Metrics::Processor.new + + metrics_processor.start routes.draw_routes end def call(env) return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze + metrics = @metrics_processor.create access_control = AccessControl.new(env) factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) env_params = params(env) raise NotFound unless factory + metrics.name_after(factory) - filtered = factory.filter_params(env_params) - service = factory.new(access_control, filtered.merge(params)) - result = service.run + filtered = factory.filter_params(env_params) + service = factory.new(access_control, filtered.merge(params)) + + metrics.tick(:prepare) + result = service.run + metrics.tick(:service) env_params.each_key { |key| result.ignored_param(key, reason: "not whitelisted".freeze) unless filtered.include?(key) } - render(result, env_params, env) + response = render(result, env_params, env) + + metrics.tick(:renderer) + metrics.success(status: response[0]) + response rescue Error => error - result = Result.new(access_control, :error, error) - V3.response(result.render(env_params, env), {}, status: error.status) + metrics.tick(:service) + + result = Result.new(access_control, :error, error) + response = V3.response(result.render(env_params, env), {}, status: error.status) + + metrics.tick(:rendered) + metrics.failure(status: error.status) + + response end def render(result, env_params, env) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ced5e7af..5161936e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -3,6 +3,7 @@ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = ENV['ENV'] = 'test' require 'support/coverage' require 'rspec' +require 'rspec/its' require 'database_cleaner' require 'sinatra/test_helpers' require 'logger' diff --git a/spec/v3/metrics_spec.rb b/spec/v3/metrics_spec.rb new file mode 100644 index 00000000..a5afe054 --- /dev/null +++ b/spec/v3/metrics_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe Travis::API::V3::Metrics do + class TestProcessor + attr_reader :times, :marks, :queue + + def initialize + @queue = [] + @times = [] + @marks = [] + end + + def time(*args) + times << args + end + + def mark(name) + marks << name + end + end + + subject(:processor) { TestProcessor.new } + let(:metric) { described_class.new(processor, time: Time.at(0)) } + + before do + metric.name_after(Travis::API::V3::Services::Branch::Find) + metric.tick(:example, time: Time.at(10)) + metric.tick(:other_example, time: Time.at(15)) + metric.success(time: Time.at(25)) + metric.process(processor) + end + + its(:queue) { should be == [metric] } + + its(:times) { should be == [ + ["branch.find.example", 10.0], + ["branch.find.other_example", 5.0], + ["branch.find.overall", 25.0] + ] } + + its(:marks) { should be == ["status.200", "branch.find.success"] } +end From efab4be6de172e4ba325f57454669da27fb16ded Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:03:09 +0100 Subject: [PATCH 04/88] comment out schema env definition --- Rakefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Rakefile b/Rakefile index 9a143017..58566cbf 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,13 @@ require 'bundler/setup' -require 'travis' +# require 'travis' require 'travis/engine' -begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production -end +# begin +# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) +# require 'micro_migrations' +# rescue LoadError +# # we can't load micro migrations on production +# end begin require 'rspec/core/rake_task' From 4833f4173214389419227674eddc9f8134c943ce Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:08:31 +0100 Subject: [PATCH 05/88] use sql schema --- Gemfile | 1 + Gemfile.lock | 10 ++++++++++ Rakefile | 17 ++++++++++------- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 9893ebf5..bf2e564b 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ group :test do gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' + gem 'travis-migrations', github: 'travis-ci/travis-migrations' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 09ee17b3..a340c150 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -72,6 +72,12 @@ GIT travis-config (~> 0.1.0) virtus (~> 1.0.0) +GIT + remote: git://github.com/travis-ci/travis-migrations.git + revision: d6105ad77303bcd04d1e376070a11dbc8b3077c7 + specs: + travis-migrations (0.0.1) + GIT remote: git://github.com/travis-ci/travis-sidekiqs.git revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23 @@ -376,8 +382,12 @@ DEPENDENCIES travis-api! travis-config (~> 0.1.0) travis-core! + travis-migrations! travis-sidekiqs! travis-support! travis-yaml! unicorn yard-sinatra! + +BUNDLED WITH + 1.10.6 diff --git a/Rakefile b/Rakefile index 58566cbf..016d761c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,16 @@ require 'bundler/setup' -# require 'travis' +require 'travis/migrations' require 'travis/engine' -# begin -# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) -# require 'micro_migrations' -# rescue LoadError -# # we can't load micro migrations on production -# end +ActiveRecord::Base.schema_format = :sql + +begin + ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + require 'micro_migrations' +rescue LoadError + # we can't load micro migrations on production +end + begin require 'rspec/core/rake_task' From 5353ed027e48d8e7d1908dacf55477fb603d4a07 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:12:15 +0100 Subject: [PATCH 06/88] add require travis --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index 016d761c..2eb5bee3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,5 @@ require 'bundler/setup' +require 'travis' require 'travis/migrations' require 'travis/engine' From 5e94e7436baa2def899b450255445d9ac812f624 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:23:37 +0100 Subject: [PATCH 07/88] omit structure dump unless dev env --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 2eb5bee3..b7e625dc 100644 --- a/Rakefile +++ b/Rakefile @@ -4,6 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql +Rake::Task["db:structure:dump"].clear unless Rails.env.development? begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -21,6 +22,7 @@ rescue LoadError warn "could not load rspec" end + desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From 8d98f39e792e86296a4f4c2b09b9d74024eae4ab Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:27:35 +0100 Subject: [PATCH 08/88] fix struture dump syntax --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index b7e625dc..3cd431ac 100644 --- a/Rakefile +++ b/Rakefile @@ -4,7 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear unless Rails.env.development? +Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) From 6b230828e68ae77d7503930edfffe958e5e5001d Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:32:40 +0100 Subject: [PATCH 09/88] move require micro-migrations to top of file --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 3cd431ac..be8fc7c7 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,6 @@ require 'bundler/setup' require 'travis' +require 'micro_migrations' require 'travis/migrations' require 'travis/engine' @@ -8,7 +9,6 @@ Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) - require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end From 66e63897a338de1a9a0886ad7d0200e44b1a9765 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:45:27 +0100 Subject: [PATCH 10/88] move structure dump --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index be8fc7c7..f97f5e20 100644 --- a/Rakefile +++ b/Rakefile @@ -5,7 +5,6 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -22,6 +21,7 @@ rescue LoadError warn "could not load rspec" end +Rake::Task["db:structure:dump"].clear desc "generate gemspec" task 'travis-api.gemspec' do From 6e8606c01ee673b0eefa835b4bb640a027524afc Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 20:53:04 +0100 Subject: [PATCH 11/88] restore to previous --- Rakefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Rakefile b/Rakefile index f97f5e20..9a143017 100644 --- a/Rakefile +++ b/Rakefile @@ -1,18 +1,14 @@ require 'bundler/setup' require 'travis' -require 'micro_migrations' -require 'travis/migrations' require 'travis/engine' -ActiveRecord::Base.schema_format = :sql - begin - ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end - begin require 'rspec/core/rake_task' RSpec::Core::RakeTask.new @@ -21,8 +17,6 @@ rescue LoadError warn "could not load rspec" end -Rake::Task["db:structure:dump"].clear - desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From b82ca5e438ce27b4a94af496e280a2bb1a5bebfa Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:10:21 +0100 Subject: [PATCH 12/88] update before script --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 08c1c58f..faa2b435 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,9 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - RAILS_ENV=test + - createdb travis_test + - bundle exec rake db:migrate --trace + +script: + - bundle exec rspec spec From 6f160912078b79a6b4ebc2cebf91bab824a69b61 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:13:12 +0100 Subject: [PATCH 13/88] update before script --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index faa2b435..c0470291 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ services: before_script: - RAILS_ENV=test - createdb travis_test + - createdb travis_pro_test - bundle exec rake db:migrate --trace script: From 172156da59c6c2c5f08fbe0374707319a93b1247 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:17:12 +0100 Subject: [PATCH 14/88] remove schema --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 9a143017..c4af9f66 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'travis' require 'travis/engine' begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) require 'micro_migrations' rescue LoadError # we can't load micro migrations on production From c0853547063f8bf1adc9be006849722b7242bae3 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:27:20 +0100 Subject: [PATCH 15/88] new rakefile --- .travis.yml | 5 +- Rakefile | 132 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 94 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0470291..58812bc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,7 @@ services: - redis before_script: - - RAILS_ENV=test - - createdb travis_test - - createdb travis_pro_test - - bundle exec rake db:migrate --trace + - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' script: - bundle exec rspec spec diff --git a/Rakefile b/Rakefile index c4af9f66..095cccec 100644 --- a/Rakefile +++ b/Rakefile @@ -1,47 +1,101 @@ +require 'rspec/core/rake_task' require 'bundler/setup' +require 'micro_migrations' require 'travis' -require 'travis/engine' -begin - # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production +ActiveRecord::Base.schema_format = :sql +Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] + +desc 'Run specs' +RSpec::Core::RakeTask.new do |t| + t.pattern = './spec/**/*_spec.rb' end -begin - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new - task default: :spec -rescue LoadError - warn "could not load rspec" -end - -desc "generate gemspec" -task 'travis-api.gemspec' do - content = File.read 'travis-api.gemspec' - - fields = { - authors: `git shortlog -sn`.scan(/[^\d\s].*/), - email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), - files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } - } - - fields.each do |field, values| - updated = " s.#{field} = [" - updated << values.map { |v| "\n %p" % v }.join(',') - updated << "\n ]" - content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) - end - - File.open('travis-api.gemspec', 'w') { |f| f << content } -end - -task default: 'travis-api.gemspec' - -tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) -Dir.glob(tasks_path).each { |r| import r } - +module ActiveRecord + class Migration + class << self + attr_accessor :disable_ddl_transaction + end + + # Disable DDL transactions for this migration. + def self.disable_ddl_transaction! + @disable_ddl_transaction = true + end + + def disable_ddl_transaction # :nodoc: + self.class.disable_ddl_transaction + end + end + + class Migrator + def use_transaction?(migration) + !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? + end + + def ddl_transaction(migration, &block) + if use_transaction?(migration) + Base.transaction { block.call } + else + block.call + end + end + + def migrate(&block) + current = migrations.detect { |m| m.version == current_version } + target = migrations.detect { |m| m.version == @target_version } + + if target.nil? && @target_version && @target_version > 0 + raise UnknownMigrationVersionError.new(@target_version) + end + + start = up? ? 0 : (migrations.index(current) || 0) + finish = migrations.index(target) || migrations.size - 1 + runnable = migrations[start..finish] + + # skip the last migration if we're headed down, but not ALL the way down + runnable.pop if down? && target + + ran = [] + runnable.each do |migration| + if block && !block.call(migration) + next + end + + Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger + + seen = migrated.include?(migration.version.to_i) + + # On our way up, we skip migrating the ones we've already migrated + next if up? && seen + + # On our way down, we skip reverting the ones we've never migrated + if down? && !seen + migration.announce 'never migrated, skipping'; migration.write + next + end + + begin + ddl_transaction(migration) do + migration.migrate(@direction) + record_version_state_after_migrating(migration.version) + end + ran << migration + rescue => e + canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" + raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace + end + end + ran + end + end + + class MigrationProxy + delegate :disable_ddl_transaction, to: :migration + end +end + +task :default => :spec + module ActiveRecord class Migration class << self From cafb078c10e3210293b2445a36f11a133026b214 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:36:32 +0100 Subject: [PATCH 16/88] add structure dump override --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 095cccec..f087e423 100644 --- a/Rakefile +++ b/Rakefile @@ -6,6 +6,8 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] + desc 'Run specs' RSpec::Core::RakeTask.new do |t| t.pattern = './spec/**/*_spec.rb' From 979201794f28845e7cbdf45fe8e6b2f915420654 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:41:40 +0100 Subject: [PATCH 17/88] fix dump override --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index f087e423..624d3300 100644 --- a/Rakefile +++ b/Rakefile @@ -6,7 +6,7 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] -Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] +Rake::Task["db:structure:dump"].clear unless Rails.env.development? desc 'Run specs' RSpec::Core::RakeTask.new do |t| From b16ae846ad409c5b00c2618a246d4d131c993338 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:49:17 +0100 Subject: [PATCH 18/88] try new rakefile --- Rakefile | 188 +++---------------------------------------------------- 1 file changed, 10 insertions(+), 178 deletions(-) diff --git a/Rakefile b/Rakefile index 624d3300..e6eecf07 100644 --- a/Rakefile +++ b/Rakefile @@ -1,182 +1,14 @@ -require 'rspec/core/rake_task' -require 'bundler/setup' -require 'micro_migrations' -require 'travis' +require 'rake' +require 'travis/migrations' -ActiveRecord::Base.schema_format = :sql -Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +task default: :spec -Rake::Task["db:structure:dump"].clear unless Rails.env.development? - -desc 'Run specs' -RSpec::Core::RakeTask.new do |t| - t.pattern = './spec/**/*_spec.rb' -end - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration - end -end - -task :default => :spec - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration +namespace :db do + desc 'Create the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh 'mkdir spec/support/db' + sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" + sh 'psql -q travis_test < spec/support/db/structure.sql' end end From 54e2ca0300130e06ff0bdd13c06f64dd0f4a5e3c Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:52:02 +0100 Subject: [PATCH 19/88] remove db:migrate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58812bc5..3f371b9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - 'RAILS_ENV=test bundle exec rake db:create --trace' script: - bundle exec rspec spec From 8d1cd15ee23a39546d2d2e6c5a3629c42707a88b Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:59:35 +0100 Subject: [PATCH 20/88] restore original db config --- config/database.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/database.yml b/config/database.yml index 53e59a34..97ba5e01 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,8 +15,8 @@ production: development: <<: *defaults - database: travis_pro_development + database: travis_development test: <<: *defaults - database: travis_pro_test + database: travis_test From 079cec3467eda24d937e8ebdc4f236e5dd9c0a27 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 5 Feb 2016 02:51:30 +0100 Subject: [PATCH 21/88] reinstate rake tasks that are still needed --- Rakefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Rakefile b/Rakefile index e6eecf07..224ae1fb 100644 --- a/Rakefile +++ b/Rakefile @@ -12,3 +12,28 @@ namespace :db do sh 'psql -q travis_test < spec/support/db/structure.sql' end end + +desc "generate gemspec" +task 'travis-api.gemspec' do + content = File.read 'travis-api.gemspec' + + fields = { + authors: `git shortlog -sn`.scan(/[^\d\s].*/), + email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), + files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } + } + + fields.each do |field, values| + updated = " s.#{field} = [" + updated << values.map { |v| "\n %p" % v }.join(',') + updated << "\n ]" + content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) + end + + File.open('travis-api.gemspec', 'w') { |f| f << content } + end + + task default: 'travis-api.gemspec' + + tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) + Dir.glob(tasks_path).each { |r| import r } From 277562e51c979c66b4e63e039549bb55d65c09c0 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 11 Feb 2016 10:13:05 +0100 Subject: [PATCH 22/88] add db creation and migration for development, update readme --- README.md | 8 ++++---- Rakefile | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b72bb7c8..9b02729f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the app running on https://api.travis-ci.org/ 1. PostgreSQL 9.3 or higher 1. Redis 1. RabbitMQ -1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. +1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. ## Installation @@ -17,8 +17,8 @@ This is the app running on https://api.travis-ci.org/ ### Database setup -1. `rake db:create db:migrate` -2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' +1. `bundle exec rake db:create` +2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): ```sh-session cd .. @@ -31,7 +31,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development Repeat the database steps for `RAILS_ENV=test`. ```sh-session -RAILS_ENV=test rake db:create db:structure:load +RAILS_ENV=test bundle exec rake db:create pushd ../travis-logs RAILS_ENV=test rvm jruby do bundle exec rake db:migrate psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test diff --git a/Rakefile b/Rakefile index 224ae1fb..93c7d171 100644 --- a/Rakefile +++ b/Rakefile @@ -4,12 +4,18 @@ require 'travis/migrations' task default: :spec namespace :db do - desc 'Create the test database' - task :create do - sh 'createdb travis_test' rescue nil - sh 'mkdir spec/support/db' - sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" - sh 'psql -q travis_test < spec/support/db/structure.sql' + if ENV["RAILS_ENV"] == 'test' + desc 'Create and migrate the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end + else + desc 'Create and migrate the development database' + task :create do + sh 'createdb travis_development' rescue nil + sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end end end From 693e2939092fc1ce0b1a56a1aac2a7a19772c6ad Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Wed, 16 Dec 2015 14:31:04 +0100 Subject: [PATCH 23/88] make sure params is always a hash, fixes exceptions --- Gemfile.lock | 3 --- lib/travis/api/app/helpers/accept.rb | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index dbc31374..f4f974f9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -377,6 +377,3 @@ DEPENDENCIES travis-yaml! unicorn yard-sinatra! - -BUNDLED WITH - 1.10.6 diff --git a/lib/travis/api/app/helpers/accept.rb b/lib/travis/api/app/helpers/accept.rb index 5fb41c67..56d7fd86 100644 --- a/lib/travis/api/app/helpers/accept.rb +++ b/lib/travis/api/app/helpers/accept.rb @@ -53,6 +53,8 @@ class Travis::Api::App if params params = Hash[*params.split(';').map { |p| p.scan /(#{TOKEN})=(#{TOKEN})/ }.flatten] quality = params.delete('q').to_f if params['q'] + else + params = {} end if subtype =~ HEADER_FORMAT From 8c3c4fe935bd07e63fbcf856a25abb29a82ed97d Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Wed, 16 Dec 2015 15:06:15 +0100 Subject: [PATCH 24/88] add ip whitelisting --- lib/travis/api/attack.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index dc0a247d..207396f5 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -31,6 +31,12 @@ class Rack::Attack "/auth/post_message/iframe" ] + #### + # Whitelisted IP addresses + whitelist('whitelist client requesting from redis') do |request| + Travis.redis.sismember(:api_whitelisted_ips, request.ip) + end + #### # Ban based on: IP address # Ban time: indefinite From 416fcf2d4b8adefa992785b06a6eff00fbc832b3 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Wed, 16 Dec 2015 15:10:03 +0100 Subject: [PATCH 25/88] Ruby 2.1.6 is no longer supported on Heroku --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index ce0fe1b8..e50f9b90 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org' gemspec -ruby '2.1.6' if ENV.key?('DYNO') +ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' From a66191c6829bec31e982ee99cae47e91211f65cf Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 15 Jan 2016 15:45:26 +0100 Subject: [PATCH 26/88] bump travis core --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f4f974f9..8c981f56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 3a9f6e4c14bb1eacb93609e1864c9c547a13c1a4 + revision: 96ee8c449ebe305c5c95633cea13eb88fe978abb specs: travis-core (0.0.1) actionmailer (~> 3.2.19) From 797ca9f9fac2514a937cf6b58b8770c61e8bfea9 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 22 Jan 2016 14:26:30 +0100 Subject: [PATCH 27/88] v3: add metrics --- Gemfile | 1 + Gemfile.lock | 4 ++ lib/travis/api/app.rb | 4 +- lib/travis/api/v3/metrics.rb | 103 +++++++++++++++++++++++++++++++++++ lib/travis/api/v3/router.rb | 35 +++++++++--- spec/spec_helper.rb | 1 + spec/v3/metrics_spec.rb | 42 ++++++++++++++ 7 files changed, 181 insertions(+), 9 deletions(-) create mode 100644 lib/travis/api/v3/metrics.rb create mode 100644 spec/v3/metrics_spec.rb diff --git a/Gemfile b/Gemfile index e50f9b90..9893ebf5 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,7 @@ gem 'customerio' group :test do gem 'rspec', '~> 2.13' + gem 'rspec-its' gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8c981f56..09ee17b3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -276,6 +276,9 @@ GEM rspec-core (2.99.2) rspec-expectations (2.99.2) diff-lcs (>= 1.1.3, < 2.0) + rspec-its (1.0.1) + rspec-core (>= 2.99.0.beta1) + rspec-expectations (>= 2.99.0.beta1) rspec-mocks (2.99.2) sidekiq (3.3.0) celluloid (>= 0.16.0) @@ -362,6 +365,7 @@ DEPENDENCIES rb-fsevent (~> 0.9.1) rerun rspec (~> 2.13) + rspec-its s3! sentry-raven! simplecov diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index a2763fdd..edc64310 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -122,7 +122,6 @@ module Travis::Api use Travis::Api::App::Middleware::Logging use Travis::Api::App::Middleware::ScopeCheck use Travis::Api::App::Middleware::UserAgentTracker - use Travis::Api::App::Middleware::Metriks # make sure this is below ScopeCheck so we have the token use Rack::Attack if Endpoint.production? @@ -133,6 +132,9 @@ module Travis::Api # rewrite should come after V3 hook use Travis::Api::App::Middleware::Rewrite + # v3 has its own metriks + use Travis::Api::App::Middleware::Metriks + SettingsEndpoint.subclass :env_vars if Travis.config.endpoints.ssh_key SingletonSettingsEndpoint.subclass :ssh_key diff --git a/lib/travis/api/v3/metrics.rb b/lib/travis/api/v3/metrics.rb new file mode 100644 index 00000000..6bf1bd3a --- /dev/null +++ b/lib/travis/api/v3/metrics.rb @@ -0,0 +1,103 @@ +require 'metriks' + +module Travis::API::V3 + class Metrics + class MetriksTracker + def initialize(prefix: "api.v3") + @prefx = prefix + end + + def time(name, duration) + ::Metriks.timer("#{@prefix}.#{name}").update(duration) + end + + def mark(name) + ::Metriks.meter("#{@prefix}.#{name}").mark + end + end + + class Processor + attr_reader :queue, :tracker + + def initialize(queue_size: 1000, tracker: MetriksTracker.new) + @tracker = tracker + @queue = queue_size ? ::SizedQueue.new(queue_size) : ::Queue.new + end + + def create(**options) + Metrics.new(self, **options) + end + + def start + Thread.new { loop { process(queue.pop) } } + end + + def process(metrics) + metrics.process(tracker) + rescue Exception => e + $stderr.puts e.message, e.backtrace + end + end + + def initialize(processor, time: Time.now) + @processor = processor + @start_time = time + @name_after = nil + @ticks = [] + @success = nil + @name = "unknown".freeze + end + + def tick(event, time: Time.now) + @ticks << [event, time] + self + end + + def success(**options) + finish(true, **options) + end + + def failure(**options) + finish(false, **options) + end + + def name_after(factory) + @name = nil + @name_after = factory + self + end + + def finish(success, time: Time.now, status: nil) + @success = !!success + @status = status + @status ||= success ? 200 : 500 + @end_time = time + @processor.queue << self + self + end + + def name + @name ||= @name_after.name[/[^:]+::[^:]+$/].underscore.tr(?/.freeze, ?..freeze) + end + + def process(tracker) + tracker.mark("status.#{@status}") + + if @success + process_ticks(tracker) + tracker.time("#{name}.overall", @end_time - @start_time) + tracker.mark("#{name}.success") + else + tracker.mark("#{name}.failure") + end + end + + def process_ticks(tracker) + start = @start_time + @ticks.each do |event, time| + tracker.time("#{name}.#{event}", time - start) + start = time + end + end + end +end diff --git a/lib/travis/api/v3/router.rb b/lib/travis/api/v3/router.rb index 8e39fc15..573c7c85 100644 --- a/lib/travis/api/v3/router.rb +++ b/lib/travis/api/v3/router.rb @@ -1,30 +1,49 @@ module Travis::API::V3 class Router include Travis::API::V3 - attr_accessor :routes + attr_accessor :routes, :metrics_processor def initialize(routes = Routes) - @routes = routes + @routes = routes + @metrics_processor = Metrics::Processor.new + + metrics_processor.start routes.draw_routes end def call(env) return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze + metrics = @metrics_processor.create access_control = AccessControl.new(env) factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) env_params = params(env) raise NotFound unless factory + metrics.name_after(factory) - filtered = factory.filter_params(env_params) - service = factory.new(access_control, filtered.merge(params)) - result = service.run + filtered = factory.filter_params(env_params) + service = factory.new(access_control, filtered.merge(params)) + + metrics.tick(:prepare) + result = service.run + metrics.tick(:service) env_params.each_key { |key| result.ignored_param(key, reason: "not whitelisted".freeze) unless filtered.include?(key) } - render(result, env_params, env) + response = render(result, env_params, env) + + metrics.tick(:renderer) + metrics.success(status: response[0]) + response rescue Error => error - result = Result.new(access_control, :error, error) - V3.response(result.render(env_params, env), {}, status: error.status) + metrics.tick(:service) + + result = Result.new(access_control, :error, error) + response = V3.response(result.render(env_params, env), {}, status: error.status) + + metrics.tick(:rendered) + metrics.failure(status: error.status) + + response end def render(result, env_params, env) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ced5e7af..5161936e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -3,6 +3,7 @@ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = ENV['ENV'] = 'test' require 'support/coverage' require 'rspec' +require 'rspec/its' require 'database_cleaner' require 'sinatra/test_helpers' require 'logger' diff --git a/spec/v3/metrics_spec.rb b/spec/v3/metrics_spec.rb new file mode 100644 index 00000000..a5afe054 --- /dev/null +++ b/spec/v3/metrics_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe Travis::API::V3::Metrics do + class TestProcessor + attr_reader :times, :marks, :queue + + def initialize + @queue = [] + @times = [] + @marks = [] + end + + def time(*args) + times << args + end + + def mark(name) + marks << name + end + end + + subject(:processor) { TestProcessor.new } + let(:metric) { described_class.new(processor, time: Time.at(0)) } + + before do + metric.name_after(Travis::API::V3::Services::Branch::Find) + metric.tick(:example, time: Time.at(10)) + metric.tick(:other_example, time: Time.at(15)) + metric.success(time: Time.at(25)) + metric.process(processor) + end + + its(:queue) { should be == [metric] } + + its(:times) { should be == [ + ["branch.find.example", 10.0], + ["branch.find.other_example", 5.0], + ["branch.find.overall", 25.0] + ] } + + its(:marks) { should be == ["status.200", "branch.find.success"] } +end From 5f32123891c0417ad38fedf01851e2eb3aad9551 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:03:09 +0100 Subject: [PATCH 28/88] comment out schema env definition --- Rakefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Rakefile b/Rakefile index 9a143017..58566cbf 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,13 @@ require 'bundler/setup' -require 'travis' +# require 'travis' require 'travis/engine' -begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production -end +# begin +# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) +# require 'micro_migrations' +# rescue LoadError +# # we can't load micro migrations on production +# end begin require 'rspec/core/rake_task' From 6c5c48316634570b91fe8b799a1dfb3c07b873b3 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:08:31 +0100 Subject: [PATCH 29/88] use sql schema --- Gemfile | 1 + Gemfile.lock | 10 ++++++++++ Rakefile | 17 ++++++++++------- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index 9893ebf5..bf2e564b 100644 --- a/Gemfile +++ b/Gemfile @@ -41,6 +41,7 @@ group :test do gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' + gem 'travis-migrations', github: 'travis-ci/travis-migrations' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 09ee17b3..a340c150 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -72,6 +72,12 @@ GIT travis-config (~> 0.1.0) virtus (~> 1.0.0) +GIT + remote: git://github.com/travis-ci/travis-migrations.git + revision: d6105ad77303bcd04d1e376070a11dbc8b3077c7 + specs: + travis-migrations (0.0.1) + GIT remote: git://github.com/travis-ci/travis-sidekiqs.git revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23 @@ -376,8 +382,12 @@ DEPENDENCIES travis-api! travis-config (~> 0.1.0) travis-core! + travis-migrations! travis-sidekiqs! travis-support! travis-yaml! unicorn yard-sinatra! + +BUNDLED WITH + 1.10.6 diff --git a/Rakefile b/Rakefile index 58566cbf..016d761c 100644 --- a/Rakefile +++ b/Rakefile @@ -1,13 +1,16 @@ require 'bundler/setup' -# require 'travis' +require 'travis/migrations' require 'travis/engine' -# begin -# ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) -# require 'micro_migrations' -# rescue LoadError -# # we can't load micro migrations on production -# end +ActiveRecord::Base.schema_format = :sql + +begin + ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + require 'micro_migrations' +rescue LoadError + # we can't load micro migrations on production +end + begin require 'rspec/core/rake_task' From c13cabfbf3d4052d8a7d6377156cf8123e9eb611 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:12:15 +0100 Subject: [PATCH 30/88] add require travis --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index 016d761c..2eb5bee3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,4 +1,5 @@ require 'bundler/setup' +require 'travis' require 'travis/migrations' require 'travis/engine' From 16db16269634bd5675eca90f0921ade855ae6eff Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:23:37 +0100 Subject: [PATCH 31/88] omit structure dump unless dev env --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 2eb5bee3..b7e625dc 100644 --- a/Rakefile +++ b/Rakefile @@ -4,6 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql +Rake::Task["db:structure:dump"].clear unless Rails.env.development? begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -21,6 +22,7 @@ rescue LoadError warn "could not load rspec" end + desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From 84cfacadb6177e000bf87f0b38a96f20b0276192 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:27:35 +0100 Subject: [PATCH 32/88] fix struture dump syntax --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index b7e625dc..3cd431ac 100644 --- a/Rakefile +++ b/Rakefile @@ -4,7 +4,7 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear unless Rails.env.development? +Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) From 15e3b84dca1e7f11b68d5c276a53a355852502f5 Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:32:40 +0100 Subject: [PATCH 33/88] move require micro-migrations to top of file --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 3cd431ac..be8fc7c7 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,6 @@ require 'bundler/setup' require 'travis' +require 'micro_migrations' require 'travis/migrations' require 'travis/engine' @@ -8,7 +9,6 @@ Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) - require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end From 345921a98b74a1ea1046dba60632a6c20d0b0d8d Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 3 Feb 2016 06:45:27 +0100 Subject: [PATCH 34/88] move structure dump --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index be8fc7c7..f97f5e20 100644 --- a/Rakefile +++ b/Rakefile @@ -5,7 +5,6 @@ require 'travis/migrations' require 'travis/engine' ActiveRecord::Base.schema_format = :sql -Rake::Task["db:structure:dump"].clear begin ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) @@ -22,6 +21,7 @@ rescue LoadError warn "could not load rspec" end +Rake::Task["db:structure:dump"].clear desc "generate gemspec" task 'travis-api.gemspec' do From 1fb526de7efc8a9b4ace8071b9d0b88950ae840f Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 20:53:04 +0100 Subject: [PATCH 35/88] restore to previous --- Rakefile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Rakefile b/Rakefile index f97f5e20..9a143017 100644 --- a/Rakefile +++ b/Rakefile @@ -1,18 +1,14 @@ require 'bundler/setup' require 'travis' -require 'micro_migrations' -require 'travis/migrations' require 'travis/engine' -ActiveRecord::Base.schema_format = :sql - begin - ENV['SCHEMA'] = File.expand_path('../db/migrate/structure.sql', $:.detect { |p| p.include?('travis-migrations') }) + ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + require 'micro_migrations' rescue LoadError # we can't load micro migrations on production end - begin require 'rspec/core/rake_task' RSpec::Core::RakeTask.new @@ -21,8 +17,6 @@ rescue LoadError warn "could not load rspec" end -Rake::Task["db:structure:dump"].clear - desc "generate gemspec" task 'travis-api.gemspec' do content = File.read 'travis-api.gemspec' From 179b166de4757e6dfa88c8b7c5a02e52a6278eb8 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:10:21 +0100 Subject: [PATCH 36/88] update before script --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 08c1c58f..faa2b435 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,9 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - RAILS_ENV=test + - createdb travis_test + - bundle exec rake db:migrate --trace + +script: + - bundle exec rspec spec From e2e222ab7cf2bfde7172d41673d331d7920f8fdb Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:13:12 +0100 Subject: [PATCH 37/88] update before script --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index faa2b435..c0470291 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ services: before_script: - RAILS_ENV=test - createdb travis_test + - createdb travis_pro_test - bundle exec rake db:migrate --trace script: From 14ec02da79e183cbc38e80aaa58939198d38bddf Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:17:12 +0100 Subject: [PATCH 38/88] remove schema --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 9a143017..c4af9f66 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,7 @@ require 'travis' require 'travis/engine' begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) + # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) require 'micro_migrations' rescue LoadError # we can't load micro migrations on production From 4d98ccafd64b33794294c4043ee41a19d72881b2 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:27:20 +0100 Subject: [PATCH 39/88] new rakefile --- .travis.yml | 5 +- Rakefile | 132 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 94 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index c0470291..58812bc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,10 +18,7 @@ services: - redis before_script: - - RAILS_ENV=test - - createdb travis_test - - createdb travis_pro_test - - bundle exec rake db:migrate --trace + - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' script: - bundle exec rspec spec diff --git a/Rakefile b/Rakefile index c4af9f66..095cccec 100644 --- a/Rakefile +++ b/Rakefile @@ -1,47 +1,101 @@ +require 'rspec/core/rake_task' require 'bundler/setup' +require 'micro_migrations' require 'travis' -require 'travis/engine' -begin - # ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production +ActiveRecord::Base.schema_format = :sql +Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] + +desc 'Run specs' +RSpec::Core::RakeTask.new do |t| + t.pattern = './spec/**/*_spec.rb' end -begin - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new - task default: :spec -rescue LoadError - warn "could not load rspec" -end - -desc "generate gemspec" -task 'travis-api.gemspec' do - content = File.read 'travis-api.gemspec' - - fields = { - authors: `git shortlog -sn`.scan(/[^\d\s].*/), - email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), - files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } - } - - fields.each do |field, values| - updated = " s.#{field} = [" - updated << values.map { |v| "\n %p" % v }.join(',') - updated << "\n ]" - content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) - end - - File.open('travis-api.gemspec', 'w') { |f| f << content } -end - -task default: 'travis-api.gemspec' - -tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) -Dir.glob(tasks_path).each { |r| import r } - +module ActiveRecord + class Migration + class << self + attr_accessor :disable_ddl_transaction + end + + # Disable DDL transactions for this migration. + def self.disable_ddl_transaction! + @disable_ddl_transaction = true + end + + def disable_ddl_transaction # :nodoc: + self.class.disable_ddl_transaction + end + end + + class Migrator + def use_transaction?(migration) + !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? + end + + def ddl_transaction(migration, &block) + if use_transaction?(migration) + Base.transaction { block.call } + else + block.call + end + end + + def migrate(&block) + current = migrations.detect { |m| m.version == current_version } + target = migrations.detect { |m| m.version == @target_version } + + if target.nil? && @target_version && @target_version > 0 + raise UnknownMigrationVersionError.new(@target_version) + end + + start = up? ? 0 : (migrations.index(current) || 0) + finish = migrations.index(target) || migrations.size - 1 + runnable = migrations[start..finish] + + # skip the last migration if we're headed down, but not ALL the way down + runnable.pop if down? && target + + ran = [] + runnable.each do |migration| + if block && !block.call(migration) + next + end + + Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger + + seen = migrated.include?(migration.version.to_i) + + # On our way up, we skip migrating the ones we've already migrated + next if up? && seen + + # On our way down, we skip reverting the ones we've never migrated + if down? && !seen + migration.announce 'never migrated, skipping'; migration.write + next + end + + begin + ddl_transaction(migration) do + migration.migrate(@direction) + record_version_state_after_migrating(migration.version) + end + ran << migration + rescue => e + canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" + raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace + end + end + ran + end + end + + class MigrationProxy + delegate :disable_ddl_transaction, to: :migration + end +end + +task :default => :spec + module ActiveRecord class Migration class << self From 25bd6b88b743dfaa080e29d5aae38bb238e04cfe Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:36:32 +0100 Subject: [PATCH 40/88] add structure dump override --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index 095cccec..f087e423 100644 --- a/Rakefile +++ b/Rakefile @@ -6,6 +6,8 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] + desc 'Run specs' RSpec::Core::RakeTask.new do |t| t.pattern = './spec/**/*_spec.rb' From e283140d02ebca39ec88abff47c65de0371bdd5b Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:41:40 +0100 Subject: [PATCH 41/88] fix dump override --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index f087e423..624d3300 100644 --- a/Rakefile +++ b/Rakefile @@ -6,7 +6,7 @@ require 'travis' ActiveRecord::Base.schema_format = :sql Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] -Rake::Task["db:structure:dump"].clear if ENV['RAILS_ENV=test'] +Rake::Task["db:structure:dump"].clear unless Rails.env.development? desc 'Run specs' RSpec::Core::RakeTask.new do |t| From c1d6441c7eb52a660659b375120488e21e203268 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:49:17 +0100 Subject: [PATCH 42/88] try new rakefile --- Rakefile | 188 +++---------------------------------------------------- 1 file changed, 10 insertions(+), 178 deletions(-) diff --git a/Rakefile b/Rakefile index 624d3300..e6eecf07 100644 --- a/Rakefile +++ b/Rakefile @@ -1,182 +1,14 @@ -require 'rspec/core/rake_task' -require 'bundler/setup' -require 'micro_migrations' -require 'travis' +require 'rake' +require 'travis/migrations' -ActiveRecord::Base.schema_format = :sql -Rails.application.config.paths["db/migrate"] = ["db/migrate", "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/migrate"] +task default: :spec -Rake::Task["db:structure:dump"].clear unless Rails.env.development? - -desc 'Run specs' -RSpec::Core::RakeTask.new do |t| - t.pattern = './spec/**/*_spec.rb' -end - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration - end -end - -task :default => :spec - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration +namespace :db do + desc 'Create the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh 'mkdir spec/support/db' + sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" + sh 'psql -q travis_test < spec/support/db/structure.sql' end end From 14b7159109d13e224c5928126c5103c0df1ed508 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:52:02 +0100 Subject: [PATCH 43/88] remove db:migrate --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58812bc5..3f371b9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - 'RAILS_ENV=test bundle exec rake db:create --trace' script: - bundle exec rspec spec From 9744f59fe276d9847beeeeb24f421e2f4432d2ec Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 4 Feb 2016 21:59:35 +0100 Subject: [PATCH 44/88] restore original db config --- config/database.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/database.yml b/config/database.yml index 53e59a34..97ba5e01 100644 --- a/config/database.yml +++ b/config/database.yml @@ -15,8 +15,8 @@ production: development: <<: *defaults - database: travis_pro_development + database: travis_development test: <<: *defaults - database: travis_pro_test + database: travis_test From 6ab0ec3c3a33103b7ef662eb5627a35f8fdb3d88 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 5 Feb 2016 02:51:30 +0100 Subject: [PATCH 45/88] reinstate rake tasks that are still needed --- Rakefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Rakefile b/Rakefile index e6eecf07..224ae1fb 100644 --- a/Rakefile +++ b/Rakefile @@ -12,3 +12,28 @@ namespace :db do sh 'psql -q travis_test < spec/support/db/structure.sql' end end + +desc "generate gemspec" +task 'travis-api.gemspec' do + content = File.read 'travis-api.gemspec' + + fields = { + authors: `git shortlog -sn`.scan(/[^\d\s].*/), + email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), + files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } + } + + fields.each do |field, values| + updated = " s.#{field} = [" + updated << values.map { |v| "\n %p" % v }.join(',') + updated << "\n ]" + content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) + end + + File.open('travis-api.gemspec', 'w') { |f| f << content } + end + + task default: 'travis-api.gemspec' + + tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) + Dir.glob(tasks_path).each { |r| import r } From 64f97a89d24a589828b0dbf58a417290951fb6e4 Mon Sep 17 00:00:00 2001 From: carlad Date: Thu, 11 Feb 2016 10:13:05 +0100 Subject: [PATCH 46/88] add db creation and migration for development, update readme --- README.md | 8 ++++---- Rakefile | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b72bb7c8..9b02729f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the app running on https://api.travis-ci.org/ 1. PostgreSQL 9.3 or higher 1. Redis 1. RabbitMQ -1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. +1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. ## Installation @@ -17,8 +17,8 @@ This is the app running on https://api.travis-ci.org/ ### Database setup -1. `rake db:create db:migrate` -2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' +1. `bundle exec rake db:create` +2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): ```sh-session cd .. @@ -31,7 +31,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development Repeat the database steps for `RAILS_ENV=test`. ```sh-session -RAILS_ENV=test rake db:create db:structure:load +RAILS_ENV=test bundle exec rake db:create pushd ../travis-logs RAILS_ENV=test rvm jruby do bundle exec rake db:migrate psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test diff --git a/Rakefile b/Rakefile index 224ae1fb..93c7d171 100644 --- a/Rakefile +++ b/Rakefile @@ -4,12 +4,18 @@ require 'travis/migrations' task default: :spec namespace :db do - desc 'Create the test database' - task :create do - sh 'createdb travis_test' rescue nil - sh 'mkdir spec/support/db' - sh "cp #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql spec/support/db/structure.sql" - sh 'psql -q travis_test < spec/support/db/structure.sql' + if ENV["RAILS_ENV"] == 'test' + desc 'Create and migrate the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end + else + desc 'Create and migrate the development database' + task :create do + sh 'createdb travis_development' rescue nil + sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end end end From 54572a5dede033c9f659fcd56505accf48ce532d Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 24 Feb 2016 23:42:50 +0100 Subject: [PATCH 47/88] update readme with info about db name --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9b02729f..5f6aaee5 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ This is the app running on https://api.travis-ci.org/ ### Database setup +NB detail for how `rake` sets up the database can be found in the `Rakefile`. In the `namespace :db` block you will see the database name for development is hardcoded to `travis-development`. If you are using a different configuration you will have to make your own adjustments. + 1. `bundle exec rake db:create` 2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): From ab1bfe0aeb4f2f91ac961512ba6445e7e5c05165 Mon Sep 17 00:00:00 2001 From: Lisa P Date: Thu, 25 Feb 2016 14:40:53 +0100 Subject: [PATCH 48/88] create MethodNotAllowed error --- lib/travis/api/v3/error.rb | 6 ++++++ lib/travis/api/v3/router.rb | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/travis/api/v3/error.rb b/lib/travis/api/v3/error.rb index fefd2b08..61d10882 100644 --- a/lib/travis/api/v3/error.rb +++ b/lib/travis/api/v3/error.rb @@ -35,4 +35,10 @@ module Travis::API::V3 super(message) end end + + class MethodNotAllowed < Error + def self.status + 405 + end + end end diff --git a/lib/travis/api/v3/router.rb b/lib/travis/api/v3/router.rb index 573c7c85..28085e81 100644 --- a/lib/travis/api/v3/router.rb +++ b/lib/travis/api/v3/router.rb @@ -15,8 +15,9 @@ module Travis::API::V3 return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze metrics = @metrics_processor.create access_control = AccessControl.new(env) - factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) env_params = params(env) + factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) + raise NotFound unless factory metrics.name_after(factory) From 04484271369dafac81fafbb2e1899a3e9f34bc6e Mon Sep 17 00:00:00 2001 From: Lisa P Date: Thu, 25 Feb 2016 15:06:45 +0100 Subject: [PATCH 49/88] create error in v3.rb --- lib/travis/api/v3.rb | 1 + lib/travis/api/v3/error.rb | 6 ------ spec/v3/error_handling_spec.rb | 15 +++++++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 spec/v3/error_handling_spec.rb diff --git a/lib/travis/api/v3.rb b/lib/travis/api/v3.rb index 8c8be745..dd8c2af6 100644 --- a/lib/travis/api/v3.rb +++ b/lib/travis/api/v3.rb @@ -35,6 +35,7 @@ module Travis NotImplemented = ServerError .create('request not (yet) implemented', status: 501) RequestLimitReached = ClientError .create('request limit reached for resource', status: 429) AlreadySyncing = ClientError .create('sync already in progress', status: 409) + MethodNotAllowed = ClientError .create('method not allowed', status: 405) end end end diff --git a/lib/travis/api/v3/error.rb b/lib/travis/api/v3/error.rb index 61d10882..fefd2b08 100644 --- a/lib/travis/api/v3/error.rb +++ b/lib/travis/api/v3/error.rb @@ -35,10 +35,4 @@ module Travis::API::V3 super(message) end end - - class MethodNotAllowed < Error - def self.status - 405 - end - end end diff --git a/spec/v3/error_handling_spec.rb b/spec/v3/error_handling_spec.rb new file mode 100644 index 00000000..b86bf917 --- /dev/null +++ b/spec/v3/error_handling_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' + +describe Travis::API::V3::ServiceIndex do + let(:headers) {{ }} + let(:path) { "/v3/repo/1/enable" } + let(:json) { JSON.load(response.body) } + let(:response) { get(path, {}, headers) } + let(:resources) { json.fetch('resources') } + + it "handles wrong HTTP method with 405 status" do + + response.status.should == 405 + end + +end From f319dfbf65e2c5261d254ec1c9325a6778c3445e Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Thu, 25 Feb 2016 16:06:32 -1000 Subject: [PATCH 50/88] Update travis-yaml --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 09ee17b3..29745665 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -87,7 +87,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-yaml.git - revision: 9ebe328e7546c696dd374a8cf773d93276f98e4f + revision: 032caed23af8ed1ed55e9204bb91316f3ada2f74 specs: travis-yaml (0.2.0) From 022089adc85104abca4d8ecaf18f5b7735150125 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 12:04:06 -1000 Subject: [PATCH 51/88] Implement bare-bones /jobs/:job_id/debug endpoint --- lib/travis/api/app/endpoint/jobs.rb | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/travis/api/app/endpoint/jobs.rb b/lib/travis/api/app/endpoint/jobs.rb index 9bb2fbc9..d06489f3 100644 --- a/lib/travis/api/app/endpoint/jobs.rb +++ b/lib/travis/api/app/endpoint/jobs.rb @@ -124,6 +124,13 @@ class Travis::Api::App end end + post "/:job_id/debug" do + job = service(:find_job, params).run + cfg = job.config + cfg.merge! debug_data + job.save! + end + def archive_url(path) "https://s3.amazonaws.com/#{hostname('archive')}#{path}" end @@ -131,6 +138,16 @@ class Travis::Api::App def hostname(name) "#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}" end + + def debug_data + { + debug: { + stage: 'before_install', + previous_status: 'failed', + created_by: current_user.login + } + } + end end end end From 8490e9f9cea5ddd972ad494e803d82241e875923 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 12:26:19 -1000 Subject: [PATCH 52/88] Debug output --- lib/travis/api/app/endpoint/jobs.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/travis/api/app/endpoint/jobs.rb b/lib/travis/api/app/endpoint/jobs.rb index d06489f3..64b2a154 100644 --- a/lib/travis/api/app/endpoint/jobs.rb +++ b/lib/travis/api/app/endpoint/jobs.rb @@ -125,10 +125,13 @@ class Travis::Api::App end post "/:job_id/debug" do + Travis.logger.debug "Reached endpoint" job = service(:find_job, params).run + Travis.logger.debug "found job: #{job}" cfg = job.config cfg.merge! debug_data job.save! + status 200 end def archive_url(path) From 2196acb36d2eb927bc1f3b248b3c33eb4cc87824 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 12:40:51 -1000 Subject: [PATCH 53/88] Use V3 services for debug route --- lib/travis/api/app/endpoint/jobs.rb | 10 ---------- lib/travis/api/v3/permissions/job.rb | 4 ++++ lib/travis/api/v3/routes.rb | 1 + lib/travis/api/v3/services/job/debug.rb | 19 +++++++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 lib/travis/api/v3/services/job/debug.rb diff --git a/lib/travis/api/app/endpoint/jobs.rb b/lib/travis/api/app/endpoint/jobs.rb index 64b2a154..cb01401b 100644 --- a/lib/travis/api/app/endpoint/jobs.rb +++ b/lib/travis/api/app/endpoint/jobs.rb @@ -124,16 +124,6 @@ class Travis::Api::App end end - post "/:job_id/debug" do - Travis.logger.debug "Reached endpoint" - job = service(:find_job, params).run - Travis.logger.debug "found job: #{job}" - cfg = job.config - cfg.merge! debug_data - job.save! - status 200 - end - def archive_url(path) "https://s3.amazonaws.com/#{hostname('archive')}#{path}" end diff --git a/lib/travis/api/v3/permissions/job.rb b/lib/travis/api/v3/permissions/job.rb index 3055257e..b2193a50 100644 --- a/lib/travis/api/v3/permissions/job.rb +++ b/lib/travis/api/v3/permissions/job.rb @@ -9,5 +9,9 @@ module Travis::API::V3 def restart? write? end + + def debug? + write? + end end end diff --git a/lib/travis/api/v3/routes.rb b/lib/travis/api/v3/routes.rb index 4243edb4..f70fbacd 100644 --- a/lib/travis/api/v3/routes.rb +++ b/lib/travis/api/v3/routes.rb @@ -34,6 +34,7 @@ module Travis::API::V3 post :cancel, '/cancel' post :restart, '/restart' + post :debug, '/debug' end resource :organization do diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb new file mode 100644 index 00000000..dd98a1bf --- /dev/null +++ b/lib/travis/api/v3/services/job/debug.rb @@ -0,0 +1,19 @@ +module Travis::API::V3 + class Services::Job::Debug < Service + + def run + raise LoginRequired unless access_control.logged_in? or access_control.full_access? + raise NotFound unless job = find(:job) + access_control.permissions(job).debug! + + Travis.logger.debug "Reached endpoint" + job = service(:find_job, params).run + Travis.logger.debug "found job: #{job}" + cfg = job.config + cfg.merge! debug_data + job.save! + + accepted(job: job, state_change: :created) + end + end +end From 1ee5421d3fd88612ee59a74e69480bc2bd0aeec4 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 12:44:17 -1000 Subject: [PATCH 54/88] Fix up Debug service --- lib/travis/api/v3/services/job/debug.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index dd98a1bf..ef6ff7fd 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -6,14 +6,20 @@ module Travis::API::V3 raise NotFound unless job = find(:job) access_control.permissions(job).debug! - Travis.logger.debug "Reached endpoint" - job = service(:find_job, params).run - Travis.logger.debug "found job: #{job}" - cfg = job.config - cfg.merge! debug_data + job.config.merge! debug_data job.save! accepted(job: job, state_change: :created) end + + def debug_data + { + debug: { + stage: 'before_install', + previous_status: 'failed', + created_by: access_control.user + } + } + end end end From e6d39c2a529df46b1c9324969866bc278f8254f1 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 14:24:09 -1000 Subject: [PATCH 55/88] Restart job via V3 query --- lib/travis/api/v3/services/job/debug.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index ef6ff7fd..3c4686cd 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -9,6 +9,7 @@ module Travis::API::V3 job.config.merge! debug_data job.save! + query.restart(access_control.user) accepted(job: job, state_change: :created) end @@ -17,7 +18,7 @@ module Travis::API::V3 debug: { stage: 'before_install', previous_status: 'failed', - created_by: access_control.user + created_by: access_control.user.login } } end From 80cb9455577fdec5bb9e343e84ded357b6feb6d5 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 14:27:57 -1000 Subject: [PATCH 56/88] Remove superfulous method --- lib/travis/api/app/endpoint/jobs.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/travis/api/app/endpoint/jobs.rb b/lib/travis/api/app/endpoint/jobs.rb index cb01401b..9bb2fbc9 100644 --- a/lib/travis/api/app/endpoint/jobs.rb +++ b/lib/travis/api/app/endpoint/jobs.rb @@ -131,16 +131,6 @@ class Travis::Api::App def hostname(name) "#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}" end - - def debug_data - { - debug: { - stage: 'before_install', - previous_status: 'failed', - created_by: current_user.login - } - } - end end end end From 89487c09cab282ef761e0d61ae5ee1802843f38a Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 14:42:34 -1000 Subject: [PATCH 57/88] Fix specs to account for new 'debug' permissions --- spec/v3/services/job/find_spec.rb | 6 +- spec/v3/services/jobs/find_spec.rb | 520 +++++++++++++++-------------- 2 files changed, 268 insertions(+), 258 deletions(-) diff --git a/spec/v3/services/job/find_spec.rb b/spec/v3/services/job/find_spec.rb index bf366386..cc2e6967 100644 --- a/spec/v3/services/job/find_spec.rb +++ b/spec/v3/services/job/find_spec.rb @@ -23,7 +23,8 @@ describe Travis::API::V3::Services::Job::Find do "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => job.id, "number" => job.number, "state" => job.state, @@ -93,7 +94,8 @@ describe Travis::API::V3::Services::Job::Find do "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => job.id, "number" => job.number, "state" => job.state, diff --git a/spec/v3/services/jobs/find_spec.rb b/spec/v3/services/jobs/find_spec.rb index f7b4c851..12ffa4fa 100644 --- a/spec/v3/services/jobs/find_spec.rb +++ b/spec/v3/services/jobs/find_spec.rb @@ -21,7 +21,8 @@ describe Travis::API::V3::Services::Jobs::Find do "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[0].id, "number" => "#{jobs[0].number}", "state" => "configured", @@ -30,45 +31,46 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}}, + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, {"@type" => "job", "@href" => "/v3/job/#{jobs[1].id}", "@representation" => "standard", "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[1].id, "number" => "#{jobs[1].number}", "state" => "configured", @@ -77,45 +79,46 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}}, + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, {"@type" => "job", "@href" => "/v3/job/#{jobs[2].id}", "@representation" => "standard", "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[2].id, "number" => "#{jobs[2].number}", "state" => "configured", @@ -124,45 +127,46 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}}, + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, {"@type" => "job", "@href" => "/v3/job/#{jobs[3].id}", "@representation" => "standard", "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[3].id, "number" => "#{jobs[3].number}", "state" => "configured", @@ -171,38 +175,38 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}} + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}} ] } } @@ -227,7 +231,8 @@ describe Travis::API::V3::Services::Jobs::Find do "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[0].id, "number" => "#{jobs[0].number}", "state" => "configured", @@ -236,45 +241,46 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}}, + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, {"@type" => "job", "@href" => "/v3/job/#{jobs[1].id}", "@representation" => "standard", "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[1].id, "number" => "#{jobs[1].number}", "state" => "configured", @@ -283,45 +289,46 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}}, + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, {"@type" => "job", "@href" => "/v3/job/#{jobs[2].id}", "@representation" => "standard", "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[2].id, "number" => "#{jobs[2].number}", "state" => "configured", @@ -330,45 +337,46 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}}, + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}}, {"@type" => "job", "@href" => "/v3/job/#{jobs[3].id}", "@representation" => "standard", "@permissions" => { "read" => true, "cancel" => false, - "restart" => false }, + "restart" => false, + "debug" => false }, "id" => jobs[3].id, "number" => "#{jobs[3].number}", "state" => "configured", @@ -377,38 +385,38 @@ describe Travis::API::V3::Services::Jobs::Find do "build" => { "@type" => "build", "@href" => "/v3/build/#{build.id}", - "@representation"=> "minimal", - "id" => build.id, - "number" => build.number, - "state" => "configured", - "duration" => nil, - "event_type" => "push", - "previous_state" => "passed", - "started_at" => "2010-11-12T13:00:00Z", - "finished_at" => nil}, - "queue" => "builds.linux", - "repository" =>{ - "@type" => "repository", - "@href" => "/v3/repo/1", - "@representation"=>"minimal", - "id" => repo.id, + "@representation"=> "minimal", + "id" => build.id, + "number" => build.number, + "state" => "configured", + "duration" => nil, + "event_type" => "push", + "previous_state" => "passed", + "started_at" => "2010-11-12T13:00:00Z", + "finished_at" => nil}, + "queue" => "builds.linux", + "repository" =>{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation"=>"minimal", + "id" => repo.id, "name" => "minimal", - "slug" => "svenfuchs/minimal"}, - "commit" =>{ - "@type" => "commit", - "@representation"=> "minimal", - "id" => commit.id, - "sha" => commit.commit, - "ref" => commit.ref, - "message" => commit.message, - "compare_url" => commit.compare_url, - "committed_at" =>"2010-11-12T12:55:00Z"}, - "owner" =>{ - "@type" => "user", - "@href" => "/v3/user/1", - "@representation"=> "minimal", - "id" => 1, - "login" => "svenfuchs"}} + "slug" => "svenfuchs/minimal"}, + "commit" =>{ + "@type" => "commit", + "@representation"=> "minimal", + "id" => commit.id, + "sha" => commit.commit, + "ref" => commit.ref, + "message" => commit.message, + "compare_url" => commit.compare_url, + "committed_at" =>"2010-11-12T12:55:00Z"}, + "owner" =>{ + "@type" => "user", + "@href" => "/v3/user/1", + "@representation"=> "minimal", + "id" => 1, + "login" => "svenfuchs"}} ] } } From 82b95440b29a61612a2778332efbf92cbbe5cf1f Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 29 Feb 2016 16:15:01 -1000 Subject: [PATCH 58/88] =?UTF-8?q?Fix=20previous=5Fstatus=20=E2=86=92=20pre?= =?UTF-8?q?vious=5Fstate,=20and=20set=20value=20correctly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/travis/api/v3/services/job/debug.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index 3c4686cd..bd3ea411 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -17,7 +17,7 @@ module Travis::API::V3 { debug: { stage: 'before_install', - previous_status: 'failed', + previous_state: job.state, created_by: access_control.user.login } } From b6b1d12f48baee5bba259fcabc7e5cc46c4da54e Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Wed, 2 Mar 2016 15:14:27 +0100 Subject: [PATCH 59/88] Hardcode database pool size for sidekiq For some reason pool is set to 1 and I can't find the source of this setting, so for now, just to fix the immediate problem I'm setting it manually in sidekiq.rb --- lib/travis/sidekiq.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index ef5aa91c..4d228473 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -7,6 +7,7 @@ require 'travis/api/workers/job_cancellation' require 'travis/api/workers/job_restart' require 'travis/support/amqp' +Travis.config.database[:pool] = 5 Travis::Database.connect if Travis.config.logs_database From fa8520eb2520469a005930af67cd22a85964584b Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Wed, 2 Mar 2016 16:14:01 +0100 Subject: [PATCH 60/88] Revert "Hardcode database pool size for sidekiq" This reverts commit b6b1d12f48baee5bba259fcabc7e5cc46c4da54e. The pool size can be set with DATABASE_POOL_SIZE ENV var on heroku. I set it to 5, so it's ok, to revert this change. --- lib/travis/sidekiq.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index 4d228473..ef5aa91c 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -7,7 +7,6 @@ require 'travis/api/workers/job_cancellation' require 'travis/api/workers/job_restart' require 'travis/support/amqp' -Travis.config.database[:pool] = 5 Travis::Database.connect if Travis.config.logs_database From d1e07f10f041d9df9623a46bff3e89bb0eec1e17 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Wed, 2 Mar 2016 09:11:12 -1000 Subject: [PATCH 61/88] Accept 'quiet' param for /job/:job_id/debug endpoint --- lib/travis/api/v3/services/job/debug.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index bd3ea411..5dac9cc8 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -1,5 +1,6 @@ module Travis::API::V3 class Services::Job::Debug < Service + params "quiet" def run raise LoginRequired unless access_control.logged_in? or access_control.full_access? @@ -13,12 +14,13 @@ module Travis::API::V3 accepted(job: job, state_change: :created) end - def debug_data + def debug_data(j) { debug: { stage: 'before_install', - previous_state: job.state, - created_by: access_control.user.login + previous_state: j.state, + created_by: access_control.user.login, + quiet: params["quiet"] || false } } end From ca09547452c824b45d348c016add90d4ff7a2078 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Wed, 2 Mar 2016 14:44:14 -1000 Subject: [PATCH 62/88] Access job via attr_reader --- lib/travis/api/v3/services/job/debug.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index 5dac9cc8..d67b53e5 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -2,9 +2,11 @@ module Travis::API::V3 class Services::Job::Debug < Service params "quiet" + attr_reader :job + def run raise LoginRequired unless access_control.logged_in? or access_control.full_access? - raise NotFound unless job = find(:job) + raise NotFound unless @job = find(:job) access_control.permissions(job).debug! job.config.merge! debug_data @@ -14,11 +16,11 @@ module Travis::API::V3 accepted(job: job, state_change: :created) end - def debug_data(j) + def debug_data { debug: { stage: 'before_install', - previous_state: j.state, + previous_state: job.state, created_by: access_control.user.login, quiet: params["quiet"] || false } From 56ea1d666311aeacae5bb705514efd904dce9561 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Thu, 3 Mar 2016 10:12:48 -1000 Subject: [PATCH 63/88] Raise if debug feature is unavailable --- lib/travis/api/v3/services/job/debug.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index d67b53e5..e3c37ab2 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -7,6 +7,7 @@ module Travis::API::V3 def run raise LoginRequired unless access_control.logged_in? or access_control.full_access? raise NotFound unless @job = find(:job) + raise WrongCredentials unless Travis.config.debug_tools_enabled or Travis::Features.active?(:debug_tools, access_control.user) access_control.permissions(job).debug! job.config.merge! debug_data From 05e860dd35e317452903e7275f0e669f5a79a004 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Thu, 3 Mar 2016 15:21:37 -1000 Subject: [PATCH 64/88] Fix `#active?` invocation Second arg is repository --- lib/travis/api/v3/services/job/debug.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index e3c37ab2..7dd1c280 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -7,7 +7,7 @@ module Travis::API::V3 def run raise LoginRequired unless access_control.logged_in? or access_control.full_access? raise NotFound unless @job = find(:job) - raise WrongCredentials unless Travis.config.debug_tools_enabled or Travis::Features.active?(:debug_tools, access_control.user) + raise WrongCredentials unless Travis.config.debug_tools_enabled or Travis::Features.active?(:debug_tools, job.repository) access_control.permissions(job).debug! job.config.merge! debug_data From ba142b84c73a24344a47aa5b36f73591088b85e6 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Fri, 4 Mar 2016 09:04:58 -1000 Subject: [PATCH 65/88] Put debug options in debug_otions This depends on https://github.com/travis-ci/travis-migrations/pull/7 --- lib/travis/api/v3/models/job.rb | 3 ++- lib/travis/api/v3/services/job/debug.rb | 12 +++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/travis/api/v3/models/job.rb b/lib/travis/api/v3/models/job.rb index 852f515c..f8f9bfb4 100644 --- a/lib/travis/api/v3/models/job.rb +++ b/lib/travis/api/v3/models/job.rb @@ -2,12 +2,13 @@ module Travis::API::V3 class Models::Job < Model self.inheritance_column = :_type_disabled - + has_one :log, dependent: :destroy belongs_to :repository belongs_to :commit belongs_to :build, autosave: true, foreign_key: 'source_id' belongs_to :owner, polymorphic: true serialize :config + serialize :debug_options end end diff --git a/lib/travis/api/v3/services/job/debug.rb b/lib/travis/api/v3/services/job/debug.rb index 7dd1c280..63c1939c 100644 --- a/lib/travis/api/v3/services/job/debug.rb +++ b/lib/travis/api/v3/services/job/debug.rb @@ -10,7 +10,7 @@ module Travis::API::V3 raise WrongCredentials unless Travis.config.debug_tools_enabled or Travis::Features.active?(:debug_tools, job.repository) access_control.permissions(job).debug! - job.config.merge! debug_data + job.debug_options = debug_data job.save! query.restart(access_control.user) @@ -19,12 +19,10 @@ module Travis::API::V3 def debug_data { - debug: { - stage: 'before_install', - previous_state: job.state, - created_by: access_control.user.login, - quiet: params["quiet"] || false - } + stage: 'before_install', + previous_state: job.state, + created_by: access_control.user.login, + quiet: params["quiet"] || false } end end From d944fe347a5fbd0e91f1ae753ae32772cf258f0f Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 7 Mar 2016 11:51:26 -1000 Subject: [PATCH 66/88] Add /job/:job_id/debug spec --- spec/v3/services/job/debug_sepc.rb | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 spec/v3/services/job/debug_sepc.rb diff --git a/spec/v3/services/job/debug_sepc.rb b/spec/v3/services/job/debug_sepc.rb new file mode 100644 index 00000000..3e3376e9 --- /dev/null +++ b/spec/v3/services/job/debug_sepc.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe Travis::API::V3::Services::Job::Debug do + let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } + let(:sidekiq_payload) { JSON.load(Sidekiq::Client.last['args'].last[:payload]).deep_symbolize_keys } + let(:sidekiq_params) { Sidekiq::Client.last['args'].last.deep_symbolize_keys } + before { repo.requests.each(&:delete) } + + before do + Travis::Features.stubs(:owner_active?).returns(true) + @original_sidekiq = Sidekiq::Client + Sidekiq.send(:remove_const, :Client) # to avoid a warning + Sidekiq::Client = [] + end + + after do + Sidekiq.send(:remove_const, :Client) # to avoid a warning + Sidekiq::Client = @original_sidekiq + end + + describe "not authenticated" do + before { post("/v3/job/#{job.id}/debug") } + example { expect(last_response.status).to be == 403 } + example { expect(JSON.load(body)).to be == { + "@type" => "error", + "error_type" => "login_required", + "error_message" => "login required" + }} + end + +end \ No newline at end of file From 73f724c58cac86a6ecf5c3d350b266e7d62f1e06 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 7 Mar 2016 16:36:33 -1000 Subject: [PATCH 67/88] Add more specs for `/job/:job_id/debug` We now rely on https://github.com/travis-ci/travis-core/pull/519. --- Gemfile | 2 +- Gemfile.lock | 7 ++-- spec/v3/services/job/debug_sepc.rb | 61 ++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/Gemfile b/Gemfile index 9893ebf5..0a612122 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' -gem 'travis-core', github: 'travis-ci/travis-core' +gem 'travis-core', github: 'travis-ci/travis-core', branch: 'ha-feature-debug-endpoint' gem 'travis-support', github: 'travis-ci/travis-support' gem 'travis-config', '~> 0.1.0' gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs', require: nil diff --git a/Gemfile.lock b/Gemfile.lock index 29745665..06399e45 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,8 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 96ee8c449ebe305c5c95633cea13eb88fe978abb + revision: 501f31a40af589b5ea8a677e7f8b067949a322b2 + branch: ha-feature-debug-endpoint specs: travis-core (0.0.1) actionmailer (~> 3.2.19) @@ -193,9 +194,9 @@ GEM httparty (0.11.0) multi_json (~> 1.0) multi_xml (>= 0.5.2) - httpclient (2.7.0.1) + httpclient (2.7.1) i18n (0.7.0) - ice_nine (0.11.1) + ice_nine (0.11.2) jemalloc (1.0.1) journey (1.0.4) json (1.8.3) diff --git a/spec/v3/services/job/debug_sepc.rb b/spec/v3/services/job/debug_sepc.rb index 3e3376e9..03fe80fb 100644 --- a/spec/v3/services/job/debug_sepc.rb +++ b/spec/v3/services/job/debug_sepc.rb @@ -2,8 +2,12 @@ require 'spec_helper' describe Travis::API::V3::Services::Job::Debug do let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first } - let(:sidekiq_payload) { JSON.load(Sidekiq::Client.last['args'].last[:payload]).deep_symbolize_keys } - let(:sidekiq_params) { Sidekiq::Client.last['args'].last.deep_symbolize_keys } + let(:owner_type) { repo.owner_type.constantize } + let(:owner) { owner_type.find(repo.owner_id)} + let(:build) { repo.builds.last } + let(:jobs) { Travis::API::V3::Models::Build.find(build.id).jobs } + let(:job) { jobs.last } + before { repo.requests.each(&:delete) } before do @@ -11,6 +15,8 @@ describe Travis::API::V3::Services::Job::Debug do @original_sidekiq = Sidekiq::Client Sidekiq.send(:remove_const, :Client) # to avoid a warning Sidekiq::Client = [] + + Travis.config.stubs(:debug_tools_enabled).returns true end after do @@ -18,14 +24,47 @@ describe Travis::API::V3::Services::Job::Debug do Sidekiq::Client = @original_sidekiq end - describe "not authenticated" do - before { post("/v3/job/#{job.id}/debug") } - example { expect(last_response.status).to be == 403 } - example { expect(JSON.load(body)).to be == { - "@type" => "error", - "error_type" => "login_required", - "error_message" => "login required" - }} - end + describe "#run" do + context "when unauthenticated" do + before { post("/v3/job/#{job.id}/debug") } + example { expect(last_response.status).to be == 403 } + example { expect(JSON.load(body)).to be == { + "@type" => "error", + "error_type" => "login_required", + "error_message" => "login required" + }} + end + context "when authenticated" do + let(:token) { Travis::Api::App::AccessToken.create(user: repo.owner, app_id: 1) } + let(:headers) {{ 'HTTP_AUTHORIZATION' => "token #{token}" }} + + context "without sufficient authorization" do + before { post("/v3/job/#{job.id}/debug", {}, headers) } + + example { expect(last_response.status).to be == 403 } + example { expect(JSON.load(body)).to include( + "@type" => "error", + "error_type" => "insufficient_access", + "error_message" => "operation requires debug access to job", + "resource_type" => "job", + )} + end + + context "with sufficient authorization" do + let(:params) {{}} + + before { Travis::API::V3::Models::Permission.create(repository: repo, user: repo.owner, push: true) } + before { post("/v3/job/#{job.id}/debug", {}, headers) } + + example { expect(last_response.status).to be == 202 } + + example { expect(job.reload.debug_options).to include( + stage: "before_install", + created_by: owner.login, + quiet: false + ) } + end + end + end end \ No newline at end of file From aaed416238bd88284d64e0c6c30d527d61ade8f6 Mon Sep 17 00:00:00 2001 From: Hiro Asari Date: Mon, 7 Mar 2016 16:51:35 -1000 Subject: [PATCH 68/88] Point to travis-core master branch https://github.com/travis-ci/travis-core/pull/519 has been merged, so there is no need for a special branch --- Gemfile | 2 +- Gemfile.lock | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 0a612122..9893ebf5 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' -gem 'travis-core', github: 'travis-ci/travis-core', branch: 'ha-feature-debug-endpoint' +gem 'travis-core', github: 'travis-ci/travis-core' gem 'travis-support', github: 'travis-ci/travis-support' gem 'travis-config', '~> 0.1.0' gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs', require: nil diff --git a/Gemfile.lock b/Gemfile.lock index 06399e45..dcf56948 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,8 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 501f31a40af589b5ea8a677e7f8b067949a322b2 - branch: ha-feature-debug-endpoint + revision: fb39f8af5628444e2d7f5893a9f09fde7b0796e2 specs: travis-core (0.0.1) actionmailer (~> 3.2.19) From 6324bbeef8c990fb65dada7a2f6cef5fd3910353 Mon Sep 17 00:00:00 2001 From: carlad Date: Tue, 8 Mar 2016 15:58:47 +0100 Subject: [PATCH 69/88] bump travis-migrations --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index a340c150..35ae27eb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -74,7 +74,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-migrations.git - revision: d6105ad77303bcd04d1e376070a11dbc8b3077c7 + revision: fcf6eea3e3122a7cbb857826db835de69974c54d specs: travis-migrations (0.0.1) From 3182513798b3a95e006b7a66602e5cf42bab61a3 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 11 Mar 2016 14:39:28 +0100 Subject: [PATCH 70/88] add sort_by default_branch builds.id --- lib/travis/api/v3/queries/repositories.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/v3/queries/repositories.rb b/lib/travis/api/v3/queries/repositories.rb index 6197a0c1..6605b620 100644 --- a/lib/travis/api/v3/queries/repositories.rb +++ b/lib/travis/api/v3/queries/repositories.rb @@ -1,7 +1,7 @@ module Travis::API::V3 class Queries::Repositories < Query params :active, :private, :starred, prefix: :repository - sortable_by :id, :github_id, :owner_name, :name, active: sort_condition(:active) + sortable_by :id, :github_id, :owner_name, :name, active: sort_condition(:active), default_branch: 'builds.id' def for_member(user, **options) all(user: user, **options).joins(:users).where(users: user_condition(user), invalidated_at: nil) From b784eaee82890e3e10a5b47109e4562cab8125c1 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 11 Mar 2016 14:41:24 +0100 Subject: [PATCH 71/88] add sort_by default_branch.last_build builds.id --- lib/travis/api/v3/queries/repositories.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/v3/queries/repositories.rb b/lib/travis/api/v3/queries/repositories.rb index 6605b620..0675d802 100644 --- a/lib/travis/api/v3/queries/repositories.rb +++ b/lib/travis/api/v3/queries/repositories.rb @@ -1,7 +1,7 @@ module Travis::API::V3 class Queries::Repositories < Query params :active, :private, :starred, prefix: :repository - sortable_by :id, :github_id, :owner_name, :name, active: sort_condition(:active), default_branch: 'builds.id' + sortable_by :id, :github_id, :owner_name, :name, active: sort_condition(:active), :'default_branch.last_build' => 'builds.id' def for_member(user, **options) all(user: user, **options).joins(:users).where(users: user_condition(user), invalidated_at: nil) From f512975e6ad0f660ffe4cce1cf4a6c0858863ee1 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 11 Mar 2016 15:05:46 +0100 Subject: [PATCH 72/88] change sort to builds.started_at --- lib/travis/api/v3/queries/repositories.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/v3/queries/repositories.rb b/lib/travis/api/v3/queries/repositories.rb index 0675d802..97d4cfc5 100644 --- a/lib/travis/api/v3/queries/repositories.rb +++ b/lib/travis/api/v3/queries/repositories.rb @@ -1,7 +1,7 @@ module Travis::API::V3 class Queries::Repositories < Query params :active, :private, :starred, prefix: :repository - sortable_by :id, :github_id, :owner_name, :name, active: sort_condition(:active), :'default_branch.last_build' => 'builds.id' + sortable_by :id, :github_id, :owner_name, :name, active: sort_condition(:active), :'default_branch.last_build' => 'builds.started_at' def for_member(user, **options) all(user: user, **options).joins(:users).where(users: user_condition(user), invalidated_at: nil) From 3a97ac8bfaf50a00ab9e6f3013951d357847f936 Mon Sep 17 00:00:00 2001 From: carlad Date: Fri, 11 Mar 2016 15:17:06 +0100 Subject: [PATCH 73/88] add spec for sort by default_branch.last_build --- .../services/repositories/for_owner_spec.rb | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index a6856581..02be8474 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -103,4 +103,26 @@ describe Travis::API::V3::Services::Repositories::ForOwner do example { expect(JSON.load(body)['@href']) .to be == "/v3/repos?starred=false" } example { expect(JSON.load(body)['repositories']) .to be_empty } end + + describe "sorting by default_branch.last_build" do + before { get("/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build") } + example { expect(last_response).to be_ok } + example { expect(parsed_body["@pagination"]).to be == { + "limit" => 100, + "offset" => 0, + "count" => 0, + "is_first" => true, + "is_last" => true, + "next" => nil, + "prev" => nil, + "first" => { + "@href" => "/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build", + "offset" => 0, + "limit" => 100 }, + "last" => { + "@href" =>"/v3/owner/svenfuchs/repos?limit=100&offset=-100&sort_by=default_branch.last_build", + "offset" => -100, + "limit" => 100 }} + } + end end From 5cb5fdb8d6f6a2997b5a8134e2b7c5df74a7f830 Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 12:14:10 +0100 Subject: [PATCH 74/88] update spec to create additional repo for sorting test --- .../services/repositories/for_owner_spec.rb | 79 ++++++++++++++----- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index 02be8474..972d7361 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -11,6 +11,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do before { repo.update_attribute(:private, true) } after { repo.update_attribute(:private, false) } + describe "private repository, private API, authenticated as user with access" do before { get("/v3/owner/svenfuchs/repos", {}, headers) } example { expect(last_response).to be_ok } @@ -105,24 +106,66 @@ describe Travis::API::V3::Services::Repositories::ForOwner do end describe "sorting by default_branch.last_build" do - before { get("/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build") } + before { repo2 = Travis::API::V3::Models::Repository.create(owner_name: 'svenfuchs', name: 'maximal', owner_id: 1, owner_type: "User", last_build_state: "passed", active: true, last_build_id: 1788, next_build_number: 3) } + before { get("/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build", {}, headers) } example { expect(last_response).to be_ok } - example { expect(parsed_body["@pagination"]).to be == { - "limit" => 100, - "offset" => 0, - "count" => 0, - "is_first" => true, - "is_last" => true, - "next" => nil, - "prev" => nil, - "first" => { - "@href" => "/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build", - "offset" => 0, - "limit" => 100 }, - "last" => { - "@href" =>"/v3/owner/svenfuchs/repos?limit=100&offset=-100&sort_by=default_branch.last_build", - "offset" => -100, - "limit" => 100 }} - } + example { expect(JSON.load(body)['@href']) .to be == "/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build" } + example { expect(JSON.load(body)['repositories']) .to be == [{ + "@type" => "repository", + "@href" => "/v3/repo/1", + "@representation" => "standard", + "@permissions" => { + "read" => true, + "enable" => false, + "disable" => false, + "star" => false, + "unstar" => false, + "create_request"=> false }, + "id" => 1, + "name" => "minimal", + "slug" => "svenfuchs/minimal", + "description" => nil, + "github_language" => nil, + "active" => true, + "private" => true, + "owner" => { + "@type" => "user", + "id" => 1, + "login" => "svenfuchs", + "@href" => "/v3/user/1" }, + "default_branch" => { + "@type" => "branch", + "@href" => "/v3/repo/1/branch/master", + "@representation"=>"minimal", + "name" => "master" }, + "starred" => false }, { + "@type" => "repository", + "@href" => "/v3/repo/5", + "@representation" => "standard", + "@permissions" => { + "read" => true, + "enable" => false, + "disable" => false, + "star" => false, + "unstar" => false, + "create_request"=> false }, + "id" => 5, + "name" => "maximal", + "slug" => "svenfuchs/maximal", + "description" => nil, + "github_language" => nil, + "active" => true, + "private" => false, + "owner" => { + "@type" => "user", + "id" => 1, + "login" => "svenfuchs", + "@href" => "/v3/user/1" }, + "default_branch" => { + "@type" => "branch", + "@href" => "/v3/repo/5/branch/master", + "@representation"=>"minimal", + "name" =>"master" }, + "starred"=>false}]} end end From 087788757f4f3cb06dd8b853482ffaa065bea3c6 Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 12:56:01 +0100 Subject: [PATCH 75/88] add repo.id to spec --- spec/v3/services/repositories/for_owner_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index 972d7361..858eac46 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -140,7 +140,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "name" => "master" }, "starred" => false }, { "@type" => "repository", - "@href" => "/v3/repo/5", + "@href" => "/v3/repo/#{repo.id}", "@representation" => "standard", "@permissions" => { "read" => true, @@ -149,7 +149,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "star" => false, "unstar" => false, "create_request"=> false }, - "id" => 5, + "id" => repo.id, "name" => "maximal", "slug" => "svenfuchs/maximal", "description" => nil, @@ -163,7 +163,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "@href" => "/v3/user/1" }, "default_branch" => { "@type" => "branch", - "@href" => "/v3/repo/5/branch/master", + "@href" => "/v3/repo/#{repo.id}/branch/master", "@representation"=>"minimal", "name" =>"master" }, "starred"=>false}]} From 74b43d5e8993dfef74b3e4b0fc239b7b0fb48043 Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 12:58:46 +0100 Subject: [PATCH 76/88] use repo2 --- spec/v3/services/repositories/for_owner_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index 858eac46..4aba71b7 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -140,7 +140,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "name" => "master" }, "starred" => false }, { "@type" => "repository", - "@href" => "/v3/repo/#{repo.id}", + "@href" => "/v3/repo/#{repo2.id}", "@representation" => "standard", "@permissions" => { "read" => true, @@ -149,7 +149,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "star" => false, "unstar" => false, "create_request"=> false }, - "id" => repo.id, + "id" => repo2.id, "name" => "maximal", "slug" => "svenfuchs/maximal", "description" => nil, @@ -163,7 +163,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "@href" => "/v3/user/1" }, "default_branch" => { "@type" => "branch", - "@href" => "/v3/repo/#{repo.id}/branch/master", + "@href" => "/v3/repo/#{repo2.id}/branch/master", "@representation"=>"minimal", "name" =>"master" }, "starred"=>false}]} From 156a6ebb9821b1f41b9755b3eab7397ea733fe64 Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 13:04:34 +0100 Subject: [PATCH 77/88] set repo id --- spec/v3/services/repositories/for_owner_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index 4aba71b7..fdf5491e 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -106,7 +106,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do end describe "sorting by default_branch.last_build" do - before { repo2 = Travis::API::V3::Models::Repository.create(owner_name: 'svenfuchs', name: 'maximal', owner_id: 1, owner_type: "User", last_build_state: "passed", active: true, last_build_id: 1788, next_build_number: 3) } + before { repo2 = Travis::API::V3::Models::Repository.create(id: 5, owner_name: 'svenfuchs', name: 'maximal', owner_id: 1, owner_type: "User", last_build_state: "passed", active: true, last_build_id: 1788, next_build_number: 3) } before { get("/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build", {}, headers) } example { expect(last_response).to be_ok } example { expect(JSON.load(body)['@href']) .to be == "/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build" } @@ -140,7 +140,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "name" => "master" }, "starred" => false }, { "@type" => "repository", - "@href" => "/v3/repo/#{repo2.id}", + "@href" => "/v3/repo/5", "@representation" => "standard", "@permissions" => { "read" => true, @@ -149,7 +149,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "star" => false, "unstar" => false, "create_request"=> false }, - "id" => repo2.id, + "id" => 5, "name" => "maximal", "slug" => "svenfuchs/maximal", "description" => nil, @@ -163,7 +163,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "@href" => "/v3/user/1" }, "default_branch" => { "@type" => "branch", - "@href" => "/v3/repo/#{repo2.id}/branch/master", + "@href" => "/v3/repo/5/branch/master", "@representation"=>"minimal", "name" =>"master" }, "starred"=>false}]} From 643ca01ee5c08af8a69069b068d5dcd11d42b6df Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 13:24:01 +0100 Subject: [PATCH 78/88] user repo2 again --- spec/v3/services/repositories/for_owner_spec.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spec/v3/services/repositories/for_owner_spec.rb b/spec/v3/services/repositories/for_owner_spec.rb index fdf5491e..0c9ac6ab 100644 --- a/spec/v3/services/repositories/for_owner_spec.rb +++ b/spec/v3/services/repositories/for_owner_spec.rb @@ -106,7 +106,8 @@ describe Travis::API::V3::Services::Repositories::ForOwner do end describe "sorting by default_branch.last_build" do - before { repo2 = Travis::API::V3::Models::Repository.create(id: 5, owner_name: 'svenfuchs', name: 'maximal', owner_id: 1, owner_type: "User", last_build_state: "passed", active: true, last_build_id: 1788, next_build_number: 3) } + let(:repo2) { Travis::API::V3::Models::Repository.create(owner_name: 'svenfuchs', name: 'maximal', owner_id: 1, owner_type: "User", last_build_state: "passed", active: true, last_build_id: 1788, next_build_number: 3) } + before { repo2.save! } before { get("/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build", {}, headers) } example { expect(last_response).to be_ok } example { expect(JSON.load(body)['@href']) .to be == "/v3/owner/svenfuchs/repos?sort_by=default_branch.last_build" } @@ -140,7 +141,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "name" => "master" }, "starred" => false }, { "@type" => "repository", - "@href" => "/v3/repo/5", + "@href" => "/v3/repo/#{repo2.id}", "@representation" => "standard", "@permissions" => { "read" => true, @@ -149,7 +150,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "star" => false, "unstar" => false, "create_request"=> false }, - "id" => 5, + "id" => repo2.id, "name" => "maximal", "slug" => "svenfuchs/maximal", "description" => nil, @@ -163,7 +164,7 @@ describe Travis::API::V3::Services::Repositories::ForOwner do "@href" => "/v3/user/1" }, "default_branch" => { "@type" => "branch", - "@href" => "/v3/repo/5/branch/master", + "@href" => "/v3/repo/#{repo2.id}/branch/master", "@representation"=>"minimal", "name" =>"master" }, "starred"=>false}]} From ac400906a1f40dfca3817fc79d7be0c78df2788d Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 14:59:33 +0100 Subject: [PATCH 79/88] bump travis-core for db statement_timeout config --- Gemfile | 2 +- Gemfile.lock | 37 +++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Gemfile b/Gemfile index bf2e564b..542a1c6e 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' -gem 'travis-core', github: 'travis-ci/travis-core' +gem 'travis-core', github: 'travis-ci/travis-core', branch: 'cd-statement-timeout' gem 'travis-support', github: 'travis-ci/travis-support' gem 'travis-config', '~> 0.1.0' gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs', require: nil diff --git a/Gemfile.lock b/Gemfile.lock index ec8ac24c..e34dd3de 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -36,9 +36,9 @@ GIT GIT remote: git://github.com/rtomayko/rack-cache.git - revision: f96febebed7700337e8c362403b081e45b8e4f13 + revision: 2d6618172c39c53dceab2e2946d87496154f3e52 specs: - rack-cache (1.5.1) + rack-cache (1.6.1) rack (>= 0.4) GIT @@ -50,7 +50,8 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: fb39f8af5628444e2d7f5893a9f09fde7b0796e2 + revision: ce489dfaf9457bd43844894c1240defea087f831 + branch: cd-statement-timeout specs: travis-core (0.0.1) actionmailer (~> 3.2.19) @@ -117,12 +118,12 @@ PATH GEM remote: https://rubygems.org/ specs: - actionmailer (3.2.22) - actionpack (= 3.2.22) + actionmailer (3.2.22.2) + actionpack (= 3.2.22.2) mail (~> 2.5.4) - actionpack (3.2.22) - activemodel (= 3.2.22) - activesupport (= 3.2.22) + actionpack (3.2.22.2) + activemodel (= 3.2.22.2) + activesupport (= 3.2.22.2) builder (~> 3.0.0) erubis (~> 2.7.0) journey (~> 1.0.4) @@ -132,15 +133,15 @@ GEM sprockets (~> 2.2.1) active_model_serializers (0.9.0) activemodel (>= 3.2) - activemodel (3.2.22) - activesupport (= 3.2.22) + activemodel (3.2.22.2) + activesupport (= 3.2.22.2) builder (~> 3.0.0) - activerecord (3.2.22) - activemodel (= 3.2.22) - activesupport (= 3.2.22) + activerecord (3.2.22.2) + activemodel (= 3.2.22.2) + activesupport (= 3.2.22.2) arel (~> 3.0.2) tzinfo (~> 0.3.29) - activesupport (3.2.22) + activesupport (3.2.22.2) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) addressable (2.4.0) @@ -151,7 +152,7 @@ GEM descendants_tracker (~> 0.0.4) ice_nine (~> 0.11.0) thread_safe (~> 0.3, >= 0.3.1) - backports (3.6.7) + backports (3.6.8) builder (3.0.4) bunny (0.8.0) celluloid (0.16.0) @@ -252,9 +253,9 @@ GEM rack rack-test (0.6.3) rack (>= 1.0) - railties (3.2.22) - actionpack (= 3.2.22) - activesupport (= 3.2.22) + railties (3.2.22.2) + actionpack (= 3.2.22.2) + activesupport (= 3.2.22.2) rack-ssl (~> 1.3.2) rake (>= 0.8.7) rdoc (~> 3.4) From e1fefaf456fd1557a39e826b5141ef49f480a861 Mon Sep 17 00:00:00 2001 From: carlad Date: Mon, 14 Mar 2016 16:29:41 +0100 Subject: [PATCH 80/88] bump travis-core statement_timeout --- Gemfile | 2 +- Gemfile.lock | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 542a1c6e..bf2e564b 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' -gem 'travis-core', github: 'travis-ci/travis-core', branch: 'cd-statement-timeout' +gem 'travis-core', github: 'travis-ci/travis-core' gem 'travis-support', github: 'travis-ci/travis-support' gem 'travis-config', '~> 0.1.0' gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs', require: nil diff --git a/Gemfile.lock b/Gemfile.lock index e34dd3de..6701537d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,8 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: ce489dfaf9457bd43844894c1240defea087f831 - branch: cd-statement-timeout + revision: 5bff6bf138d31754e38b8c148e56ba8ceab39859 specs: travis-core (0.0.1) actionmailer (~> 3.2.19) From 7e438be4cfa324b15c98222559705a86bd3bf2ec Mon Sep 17 00:00:00 2001 From: carlad Date: Wed, 16 Mar 2016 15:20:41 +0100 Subject: [PATCH 81/88] remove hard coded request path --- lib/travis/api/attack.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index 207396f5..17fcd44a 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -61,7 +61,7 @@ class Rack::Attack # Ban after: 10 POST requests within 30 seconds blacklist('spamming with POST requests') do |request| Rack::Attack::Allow2Ban.filter(request.identifier, maxretry: 10, findtime: 30.seconds, bantime: bantime(1.hour)) do - request.post? and not POST_WHITELISTED.include? '/auth/github' + request.post? and not POST_WHITELISTED.include? request.path end end From b68c7a33b03b9f8bb937aafbb9b3b4b10d5072cf Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 10:57:12 +0100 Subject: [PATCH 82/88] Allow to set sidekiq concurrency separately from web workers Unicorn, which we use to serve the API, uses forked workers that need only one connection per instance. Sidekiq on the other hand runs several instances in memory, so it needs a higher concurrency setting. This commit introduces a way to set sidekiq db pool using SIDEKIQ_DB_POOL_SIZE --- lib/travis/sidekiq.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index ef5aa91c..38b651be 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -7,6 +7,7 @@ require 'travis/api/workers/job_cancellation' require 'travis/api/workers/job_restart' require 'travis/support/amqp' +Travis.config.database[:pool] = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 Travis::Database.connect if Travis.config.logs_database From 5682502cbaa8d35a751f6179971e77a5c0cf1af4 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:14:46 +0100 Subject: [PATCH 83/88] Bump travis-core --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6701537d..3553f293 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 5bff6bf138d31754e38b8c148e56ba8ceab39859 + revision: 808cc8f32e6e1054e7231c14a36b0489b09446ff specs: travis-core (0.0.1) actionmailer (~> 3.2.19) @@ -331,7 +331,7 @@ GEM treetop (1.4.15) polyglot polyglot (>= 0.3.1) - tzinfo (0.3.46) + tzinfo (0.3.47) unicorn (4.8.3) kgio (~> 2.6) rack @@ -390,4 +390,4 @@ DEPENDENCIES yard-sinatra! BUNDLED WITH - 1.10.6 + 1.11.2 From f2c96b3eb52cede315acde77743b924106e05195 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:27:42 +0100 Subject: [PATCH 84/88] Set sidekiq pool size also for logs DB --- lib/travis/sidekiq.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index 38b651be..936c4653 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -7,7 +7,9 @@ require 'travis/api/workers/job_cancellation' require 'travis/api/workers/job_restart' require 'travis/support/amqp' -Travis.config.database[:pool] = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 +pool_size = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 +Travis.config.database[:pool] = pool_size +Travis.config.logs_database[:pool] = pool_size Travis::Database.connect if Travis.config.logs_database From 6aaa25a099a457e96001701460578777fb45b22a Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:45:46 +0100 Subject: [PATCH 85/88] Overwrite database pool size for logs database Logs database config is set in travis keychain and is set to a concurrency much higher than what we set in heroku config. I don't want to change it globally, because it may need to be different for other apps, so I'm changing it here just before we connect to the db. --- lib/travis/api/app.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index edc64310..25af74ba 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -199,6 +199,9 @@ module Travis::Api Travis::Database.connect if Travis.config.logs_database + pool_size = ENV['DATABASE_POOL_SIZE'] + Travis.config.logs_database[:pool] = pool_size if pool_size + Log.establish_connection 'logs_database' Log::Part.establish_connection 'logs_database' end From f0dcf270bc44cb8256d99ba438793dc05352b2f9 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 11:59:34 +0100 Subject: [PATCH 86/88] Always convert pool size to integer --- lib/travis/api/app.rb | 2 +- lib/travis/sidekiq.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index 25af74ba..d9719090 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -200,7 +200,7 @@ module Travis::Api if Travis.config.logs_database pool_size = ENV['DATABASE_POOL_SIZE'] - Travis.config.logs_database[:pool] = pool_size if pool_size + Travis.config.logs_database[:pool] = pool_size.to_i if pool_size Log.establish_connection 'logs_database' Log::Part.establish_connection 'logs_database' diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index 936c4653..bb9e3574 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -8,8 +8,8 @@ require 'travis/api/workers/job_restart' require 'travis/support/amqp' pool_size = ENV['SIDEKIQ_DB_POOL_SIZE'] || 5 -Travis.config.database[:pool] = pool_size -Travis.config.logs_database[:pool] = pool_size +Travis.config.database[:pool] = pool_size.to_i +Travis.config.logs_database[:pool] = pool_size.to_i Travis::Database.connect if Travis.config.logs_database From 5294f1dab25523ca0db8debd8a9c3d5697d2d939 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 13:00:29 +0100 Subject: [PATCH 87/88] Bump travis-core --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3553f293..f4305853 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 808cc8f32e6e1054e7231c14a36b0489b09446ff + revision: f7b3a76b3f39c28bb5cf7b9dc24acec13908a11a specs: travis-core (0.0.1) actionmailer (~> 3.2.19) From 8a050a468c77f2ad8c5faa99864c0bd7d61b08a0 Mon Sep 17 00:00:00 2001 From: Piotr Sarnacki Date: Thu, 17 Mar 2016 13:07:23 +0100 Subject: [PATCH 88/88] Use shared logs model to establish connection That way we will use only one connection to the logs database --- lib/travis/api/app.rb | 3 +-- lib/travis/sidekiq.rb | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index d9719090..6b72b8ce 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -202,8 +202,7 @@ module Travis::Api pool_size = ENV['DATABASE_POOL_SIZE'] Travis.config.logs_database[:pool] = pool_size.to_i if pool_size - Log.establish_connection 'logs_database' - Log::Part.establish_connection 'logs_database' + Travis::LogsModel.establish_connection 'logs_database' end end diff --git a/lib/travis/sidekiq.rb b/lib/travis/sidekiq.rb index bb9e3574..077e6055 100644 --- a/lib/travis/sidekiq.rb +++ b/lib/travis/sidekiq.rb @@ -13,8 +13,7 @@ Travis.config.logs_database[:pool] = pool_size.to_i Travis::Database.connect if Travis.config.logs_database - Log.establish_connection 'logs_database' - Log::Part.establish_connection 'logs_database' + Travis::LogsModel.establish_connection 'logs_database' end Travis::Async.enabled = true