diff --git a/.travis.yml b/.travis.yml index 08c1c58f..3f371b9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,7 @@ services: - redis before_script: - - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' + - 'RAILS_ENV=test bundle exec rake db:create --trace' + +script: + - bundle exec rspec spec diff --git a/Gemfile b/Gemfile index ce0fe1b8..bf2e564b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org' gemspec -ruby '2.1.6' if ENV.key?('DYNO') +ruby '2.1.7' if ENV.key?('DYNO') gem 's3', github: 'travis-ci/s3' @@ -37,9 +37,11 @@ gem 'customerio' group :test do gem 'rspec', '~> 2.13' + gem 'rspec-its' gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' + gem 'travis-migrations', github: 'travis-ci/travis-migrations' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index f4f974f9..c3551b4e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,7 +50,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-core.git - revision: 3a9f6e4c14bb1eacb93609e1864c9c547a13c1a4 + revision: 96ee8c449ebe305c5c95633cea13eb88fe978abb specs: travis-core (0.0.1) actionmailer (~> 3.2.19) @@ -72,6 +72,12 @@ GIT travis-config (~> 0.1.0) virtus (~> 1.0.0) +GIT + remote: git://github.com/travis-ci/travis-migrations.git + revision: fcf6eea3e3122a7cbb857826db835de69974c54d + specs: + travis-migrations (0.0.1) + GIT remote: git://github.com/travis-ci/travis-sidekiqs.git revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23 @@ -87,7 +93,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-yaml.git - revision: 9ebe328e7546c696dd374a8cf773d93276f98e4f + revision: 032caed23af8ed1ed55e9204bb91316f3ada2f74 specs: travis-yaml (0.2.0) @@ -276,6 +282,9 @@ GEM rspec-core (2.99.2) rspec-expectations (2.99.2) diff-lcs (>= 1.1.3, < 2.0) + rspec-its (1.0.1) + rspec-core (>= 2.99.0.beta1) + rspec-expectations (>= 2.99.0.beta1) rspec-mocks (2.99.2) sidekiq (3.3.0) celluloid (>= 0.16.0) @@ -362,6 +371,7 @@ DEPENDENCIES rb-fsevent (~> 0.9.1) rerun rspec (~> 2.13) + rspec-its s3! sentry-raven! simplecov @@ -372,8 +382,12 @@ DEPENDENCIES travis-api! travis-config (~> 0.1.0) travis-core! + travis-migrations! travis-sidekiqs! travis-support! travis-yaml! unicorn yard-sinatra! + +BUNDLED WITH + 1.10.6 diff --git a/README.md b/README.md index b72bb7c8..5f6aaee5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This is the app running on https://api.travis-ci.org/ 1. PostgreSQL 9.3 or higher 1. Redis 1. RabbitMQ -1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. +1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally. ## Installation @@ -17,8 +17,10 @@ This is the app running on https://api.travis-ci.org/ ### Database setup -1. `rake db:create db:migrate` -2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace' +NB detail for how `rake` sets up the database can be found in the `Rakefile`. In the `namespace :db` block you will see the database name for development is hardcoded to `travis-development`. If you are using a different configuration you will have to make your own adjustments. + +1. `bundle exec rake db:create` +2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace' 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`): ```sh-session cd .. @@ -31,7 +33,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development Repeat the database steps for `RAILS_ENV=test`. ```sh-session -RAILS_ENV=test rake db:create db:structure:load +RAILS_ENV=test bundle exec rake db:create pushd ../travis-logs RAILS_ENV=test rvm jruby do bundle exec rake db:migrate psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test diff --git a/Rakefile b/Rakefile index 866a2958..93c7d171 100644 --- a/Rakefile +++ b/Rakefile @@ -1,127 +1,45 @@ -require 'bundler/setup' -require 'travis' -require 'travis/engine' +require 'rake' +require 'travis/migrations' -begin - ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') }) - require 'micro_migrations' -rescue LoadError - # we can't load micro migrations on production -end -require 'travis' +task default: :spec -begin - require 'rspec/core/rake_task' - RSpec::Core::RakeTask.new - task default: :spec -rescue LoadError - warn "could not load rspec" +namespace :db do + if ENV["RAILS_ENV"] == 'test' + desc 'Create and migrate the test database' + task :create do + sh 'createdb travis_test' rescue nil + sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end + else + desc 'Create and migrate the development database' + task :create do + sh 'createdb travis_development' rescue nil + sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql" + end + end end desc "generate gemspec" task 'travis-api.gemspec' do - content = File.read 'travis-api.gemspec' + content = File.read 'travis-api.gemspec' - fields = { - authors: `git shortlog -sn`.scan(/[^\d\s].*/), - email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), - files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } - } + fields = { + authors: `git shortlog -sn`.scan(/[^\d\s].*/), + email: `git shortlog -sne`.scan(/[^<]+@[^>]+/), + files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ } + } - fields.each do |field, values| - updated = " s.#{field} = [" - updated << values.map { |v| "\n %p" % v }.join(',') - updated << "\n ]" - content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) - end + fields.each do |field, values| + updated = " s.#{field} = [" + updated << values.map { |v| "\n %p" % v }.join(',') + updated << "\n ]" + content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated) + end - File.open('travis-api.gemspec', 'w') { |f| f << content } -end + File.open('travis-api.gemspec', 'w') { |f| f << content } + end -task default: 'travis-api.gemspec' + task default: 'travis-api.gemspec' -tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) -Dir.glob(tasks_path).each { |r| import r } - -module ActiveRecord - class Migration - class << self - attr_accessor :disable_ddl_transaction - end - - # Disable DDL transactions for this migration. - def self.disable_ddl_transaction! - @disable_ddl_transaction = true - end - - def disable_ddl_transaction # :nodoc: - self.class.disable_ddl_transaction - end - end - - class Migrator - def use_transaction?(migration) - !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? - end - - def ddl_transaction(migration, &block) - if use_transaction?(migration) - Base.transaction { block.call } - else - block.call - end - end - - def migrate(&block) - current = migrations.detect { |m| m.version == current_version } - target = migrations.detect { |m| m.version == @target_version } - - if target.nil? && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - start = up? ? 0 : (migrations.index(current) || 0) - finish = migrations.index(target) || migrations.size - 1 - runnable = migrations[start..finish] - - # skip the last migration if we're headed down, but not ALL the way down - runnable.pop if down? && target - - ran = [] - runnable.each do |migration| - if block && !block.call(migration) - next - end - - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - seen = migrated.include?(migration.version.to_i) - - # On our way up, we skip migrating the ones we've already migrated - next if up? && seen - - # On our way down, we skip reverting the ones we've never migrated - if down? && !seen - migration.announce 'never migrated, skipping'; migration.write - next - end - - begin - ddl_transaction(migration) do - migration.migrate(@direction) - record_version_state_after_migrating(migration.version) - end - ran << migration - rescue => e - canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end - end - ran - end - end - - class MigrationProxy - delegate :disable_ddl_transaction, to: :migration - end -end + tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__) + Dir.glob(tasks_path).each { |r| import r } diff --git a/config/database.yml b/config/database.yml index 26b09327..97ba5e01 100644 --- a/config/database.yml +++ b/config/database.yml @@ -20,4 +20,3 @@ development: test: <<: *defaults database: travis_test - diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index a2763fdd..edc64310 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -122,7 +122,6 @@ module Travis::Api use Travis::Api::App::Middleware::Logging use Travis::Api::App::Middleware::ScopeCheck use Travis::Api::App::Middleware::UserAgentTracker - use Travis::Api::App::Middleware::Metriks # make sure this is below ScopeCheck so we have the token use Rack::Attack if Endpoint.production? @@ -133,6 +132,9 @@ module Travis::Api # rewrite should come after V3 hook use Travis::Api::App::Middleware::Rewrite + # v3 has its own metriks + use Travis::Api::App::Middleware::Metriks + SettingsEndpoint.subclass :env_vars if Travis.config.endpoints.ssh_key SingletonSettingsEndpoint.subclass :ssh_key diff --git a/lib/travis/api/v2/http/build.rb b/lib/travis/api/v2/http/build.rb index 9d8203b5..394c4ac8 100644 --- a/lib/travis/api/v2/http/build.rb +++ b/lib/travis/api/v2/http/build.rb @@ -17,7 +17,7 @@ module Travis def data { 'build' => build_data(build), - 'commit' => commit_data(build.commit), + 'commit' => commit_data(build.commit, build.repository), 'jobs' => options[:include_jobs] ? build.matrix.map { |job| job_data(job) } : [], 'annotations' => options[:include_jobs] ? Annotations.new(annotations(build), @options).data["annotations"] : [], } @@ -44,11 +44,12 @@ module Travis } end - def commit_data(commit) + def commit_data(commit, repository) { 'id' => commit.id, 'sha' => commit.commit, 'branch' => commit.branch, + 'branch_is_default' => branch_is_default(commit, repository), 'message' => commit.message, 'committed_at' => format_date(commit.committed_at), 'author_name' => commit.author_name, @@ -78,6 +79,10 @@ module Travis } end + def branch_is_default(commit, repository) + repository.default_branch == commit.branch + end + def annotations(build) build.matrix.map(&:annotations).flatten end diff --git a/lib/travis/api/v2/http/job.rb b/lib/travis/api/v2/http/job.rb index 4b712b40..92366474 100644 --- a/lib/travis/api/v2/http/job.rb +++ b/lib/travis/api/v2/http/job.rb @@ -15,7 +15,7 @@ module Travis def data { 'job' => job_data(job), - 'commit' => commit_data(job.commit), + 'commit' => commit_data(job.commit, job.repository), 'annotations' => Annotations.new(job.annotations, @options).data["annotations"], } end @@ -42,11 +42,12 @@ module Travis } end - def commit_data(commit) + def commit_data(commit, repository) { 'id' => commit.id, 'sha' => commit.commit, 'branch' => commit.branch, + 'branch_is_default' => branch_is_default(commit, repository), 'message' => commit.message, 'committed_at' => format_date(commit.committed_at), 'author_name' => commit.author_name, @@ -56,6 +57,10 @@ module Travis 'compare_url' => commit.compare_url, } end + + def branch_is_default(commit, repository) + repository.default_branch == commit.branch + end end end end diff --git a/lib/travis/api/v3.rb b/lib/travis/api/v3.rb index 8c8be745..dd8c2af6 100644 --- a/lib/travis/api/v3.rb +++ b/lib/travis/api/v3.rb @@ -35,6 +35,7 @@ module Travis NotImplemented = ServerError .create('request not (yet) implemented', status: 501) RequestLimitReached = ClientError .create('request limit reached for resource', status: 429) AlreadySyncing = ClientError .create('sync already in progress', status: 409) + MethodNotAllowed = ClientError .create('method not allowed', status: 405) end end end diff --git a/lib/travis/api/v3/metrics.rb b/lib/travis/api/v3/metrics.rb new file mode 100644 index 00000000..6bf1bd3a --- /dev/null +++ b/lib/travis/api/v3/metrics.rb @@ -0,0 +1,103 @@ +require 'metriks' + +module Travis::API::V3 + class Metrics + class MetriksTracker + def initialize(prefix: "api.v3") + @prefx = prefix + end + + def time(name, duration) + ::Metriks.timer("#{@prefix}.#{name}").update(duration) + end + + def mark(name) + ::Metriks.meter("#{@prefix}.#{name}").mark + end + end + + class Processor + attr_reader :queue, :tracker + + def initialize(queue_size: 1000, tracker: MetriksTracker.new) + @tracker = tracker + @queue = queue_size ? ::SizedQueue.new(queue_size) : ::Queue.new + end + + def create(**options) + Metrics.new(self, **options) + end + + def start + Thread.new { loop { process(queue.pop) } } + end + + def process(metrics) + metrics.process(tracker) + rescue Exception => e + $stderr.puts e.message, e.backtrace + end + end + + def initialize(processor, time: Time.now) + @processor = processor + @start_time = time + @name_after = nil + @ticks = [] + @success = nil + @name = "unknown".freeze + end + + def tick(event, time: Time.now) + @ticks << [event, time] + self + end + + def success(**options) + finish(true, **options) + end + + def failure(**options) + finish(false, **options) + end + + def name_after(factory) + @name = nil + @name_after = factory + self + end + + def finish(success, time: Time.now, status: nil) + @success = !!success + @status = status + @status ||= success ? 200 : 500 + @end_time = time + @processor.queue << self + self + end + + def name + @name ||= @name_after.name[/[^:]+::[^:]+$/].underscore.tr(?/.freeze, ?..freeze) + end + + def process(tracker) + tracker.mark("status.#{@status}") + + if @success + process_ticks(tracker) + tracker.time("#{name}.overall", @end_time - @start_time) + tracker.mark("#{name}.success") + else + tracker.mark("#{name}.failure") + end + end + + def process_ticks(tracker) + start = @start_time + @ticks.each do |event, time| + tracker.time("#{name}.#{event}", time - start) + start = time + end + end + end +end diff --git a/lib/travis/api/v3/router.rb b/lib/travis/api/v3/router.rb index 8e39fc15..28085e81 100644 --- a/lib/travis/api/v3/router.rb +++ b/lib/travis/api/v3/router.rb @@ -1,30 +1,50 @@ module Travis::API::V3 class Router include Travis::API::V3 - attr_accessor :routes + attr_accessor :routes, :metrics_processor def initialize(routes = Routes) - @routes = routes + @routes = routes + @metrics_processor = Metrics::Processor.new + + metrics_processor.start routes.draw_routes end def call(env) return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze + metrics = @metrics_processor.create access_control = AccessControl.new(env) - factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) env_params = params(env) + factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze]) + raise NotFound unless factory + metrics.name_after(factory) - filtered = factory.filter_params(env_params) - service = factory.new(access_control, filtered.merge(params)) - result = service.run + filtered = factory.filter_params(env_params) + service = factory.new(access_control, filtered.merge(params)) + + metrics.tick(:prepare) + result = service.run + metrics.tick(:service) env_params.each_key { |key| result.ignored_param(key, reason: "not whitelisted".freeze) unless filtered.include?(key) } - render(result, env_params, env) + response = render(result, env_params, env) + + metrics.tick(:renderer) + metrics.success(status: response[0]) + response rescue Error => error - result = Result.new(access_control, :error, error) - V3.response(result.render(env_params, env), {}, status: error.status) + metrics.tick(:service) + + result = Result.new(access_control, :error, error) + response = V3.response(result.render(env_params, env), {}, status: error.status) + + metrics.tick(:rendered) + metrics.failure(status: error.status) + + response end def render(result, env_params, env) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ced5e7af..5161936e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -3,6 +3,7 @@ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = ENV['ENV'] = 'test' require 'support/coverage' require 'rspec' +require 'rspec/its' require 'database_cleaner' require 'sinatra/test_helpers' require 'logger' diff --git a/spec/unit/api/v2/http/build_spec.rb b/spec/unit/api/v2/http/build_spec.rb index 11a6dbea..f276b2b8 100644 --- a/spec/unit/api/v2/http/build_spec.rb +++ b/spec/unit/api/v2/http/build_spec.rb @@ -29,6 +29,7 @@ describe Travis::Api::V2::Http::Build do 'id' => 1, 'sha' => '62aae5f70ceee39123ef', 'branch' => 'master', + 'branch_is_default' => true, 'message' => 'the commit message', 'compare_url' => 'https://github.com/svenfuchs/minimal/compare/master...develop', 'committed_at' => json_format_time(Time.now.utc - 1.hour), diff --git a/spec/unit/api/v2/http/job_spec.rb b/spec/unit/api/v2/http/job_spec.rb index a9211b6f..22fda7b6 100644 --- a/spec/unit/api/v2/http/job_spec.rb +++ b/spec/unit/api/v2/http/job_spec.rb @@ -31,6 +31,7 @@ describe Travis::Api::V2::Http::Job do 'sha' => '62aae5f70ceee39123ef', 'message' => 'the commit message', 'branch' => 'master', + 'branch_is_default' => true, 'message' => 'the commit message', 'committed_at' => json_format_time(Time.now.utc - 1.hour), 'committer_name' => 'Sven Fuchs', diff --git a/spec/v3/error_handling_spec.rb b/spec/v3/error_handling_spec.rb new file mode 100644 index 00000000..b86bf917 --- /dev/null +++ b/spec/v3/error_handling_spec.rb @@ -0,0 +1,15 @@ +require 'spec_helper' + +describe Travis::API::V3::ServiceIndex do + let(:headers) {{ }} + let(:path) { "/v3/repo/1/enable" } + let(:json) { JSON.load(response.body) } + let(:response) { get(path, {}, headers) } + let(:resources) { json.fetch('resources') } + + it "handles wrong HTTP method with 405 status" do + + response.status.should == 405 + end + +end diff --git a/spec/v3/metrics_spec.rb b/spec/v3/metrics_spec.rb new file mode 100644 index 00000000..a5afe054 --- /dev/null +++ b/spec/v3/metrics_spec.rb @@ -0,0 +1,42 @@ +require 'spec_helper' + +describe Travis::API::V3::Metrics do + class TestProcessor + attr_reader :times, :marks, :queue + + def initialize + @queue = [] + @times = [] + @marks = [] + end + + def time(*args) + times << args + end + + def mark(name) + marks << name + end + end + + subject(:processor) { TestProcessor.new } + let(:metric) { described_class.new(processor, time: Time.at(0)) } + + before do + metric.name_after(Travis::API::V3::Services::Branch::Find) + metric.tick(:example, time: Time.at(10)) + metric.tick(:other_example, time: Time.at(15)) + metric.success(time: Time.at(25)) + metric.process(processor) + end + + its(:queue) { should be == [metric] } + + its(:times) { should be == [ + ["branch.find.example", 10.0], + ["branch.find.other_example", 5.0], + ["branch.find.overall", 25.0] + ] } + + its(:marks) { should be == ["status.200", "branch.find.success"] } +end