Merge branch 'master' into cd-v3

.travis.yml

@@ -18,4 +18,7 @@ services:
   - redis
 
 before_script:
-  - 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace'
+  - 'RAILS_ENV=test bundle exec rake db:create --trace'
+
+script:
+  - bundle exec rspec spec

Gemfile

@@ -1,7 +1,7 @@
 source 'https://rubygems.org'
 gemspec
 
-ruby '2.1.6' if ENV.key?('DYNO')
+ruby '2.1.7' if ENV.key?('DYNO')
 
 gem 's3',              github: 'travis-ci/s3'
 
@@ -37,9 +37,11 @@ gem 'customerio'
 
 group :test do
   gem 'rspec',         '~> 2.13'
+  gem 'rspec-its'
   gem 'factory_girl',  '~> 2.4.0'
   gem 'mocha',         '~> 0.12'
   gem 'database_cleaner', '~> 0.8.0'
+  gem 'travis-migrations', github: 'travis-ci/travis-migrations'
 end
 
 group :development do

Gemfile.lock

@@ -50,7 +50,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-core.git
-  revision: 3a9f6e4c14bb1eacb93609e1864c9c547a13c1a4
+  revision: 96ee8c449ebe305c5c95633cea13eb88fe978abb
   specs:
     travis-core (0.0.1)
       actionmailer (~> 3.2.19)
@@ -72,6 +72,12 @@ GIT
       travis-config (~> 0.1.0)
       virtus (~> 1.0.0)
 
+GIT
+  remote: git://github.com/travis-ci/travis-migrations.git
+  revision: fcf6eea3e3122a7cbb857826db835de69974c54d
+  specs:
+    travis-migrations (0.0.1)
+
 GIT
   remote: git://github.com/travis-ci/travis-sidekiqs.git
   revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23
@@ -87,7 +93,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-yaml.git
-  revision: 9ebe328e7546c696dd374a8cf773d93276f98e4f
+  revision: 032caed23af8ed1ed55e9204bb91316f3ada2f74
   specs:
     travis-yaml (0.2.0)
 
@@ -276,6 +282,9 @@ GEM
     rspec-core (2.99.2)
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
+    rspec-its (1.0.1)
+      rspec-core (>= 2.99.0.beta1)
+      rspec-expectations (>= 2.99.0.beta1)
     rspec-mocks (2.99.2)
     sidekiq (3.3.0)
       celluloid (>= 0.16.0)
@@ -362,6 +371,7 @@ DEPENDENCIES
   rb-fsevent (~> 0.9.1)
   rerun
   rspec (~> 2.13)
+  rspec-its
   s3!
   sentry-raven!
   simplecov
@@ -372,8 +382,12 @@ DEPENDENCIES
   travis-api!
   travis-config (~> 0.1.0)
   travis-core!
+  travis-migrations!
   travis-sidekiqs!
   travis-support!
   travis-yaml!
   unicorn
   yard-sinatra!
+
+BUNDLED WITH
+   1.10.6

README.md

@@ -17,8 +17,10 @@ This is the app running on https://api.travis-ci.org/
 
 ### Database setup
 
-1. `rake db:create db:migrate`
+NB detail for how `rake` sets up the database can be found in the `Rakefile`. In the `namespace :db` block you will see the database name for development is hardcoded to `travis-development`. If you are using a different configuration you will have to make your own adjustments.
-2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace'
+
+1. `bundle exec rake db:create`
+2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace'
 1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`):
 ```sh-session
 cd ..
@@ -31,7 +33,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development
 
 Repeat the database steps for `RAILS_ENV=test`.
 ```sh-session
-RAILS_ENV=test rake db:create db:structure:load
+RAILS_ENV=test bundle exec rake db:create
 pushd ../travis-logs
 RAILS_ENV=test rvm jruby do bundle exec rake db:migrate
 psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test

Rakefile

@@ -1,21 +1,22 @@
-require 'bundler/setup'
+require 'rake'
-require 'travis'
+require 'travis/migrations'
-require 'travis/engine'
 
-begin
+task default: :spec
-  ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') })
-  require 'micro_migrations'
-rescue LoadError
-  # we can't load micro migrations on production
-end
-require 'travis'
 
-begin
+namespace :db do
-  require 'rspec/core/rake_task'
+  if ENV["RAILS_ENV"] == 'test'
-  RSpec::Core::RakeTask.new
+    desc 'Create and migrate the test database'
-  task default: :spec
+    task :create do
-rescue LoadError
+      sh 'createdb travis_test' rescue nil
-  warn "could not load rspec"
+      sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql"
+    end
+  else
+    desc 'Create and migrate the development database'
+    task :create do
+      sh 'createdb travis_development' rescue nil
+      sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql"
+    end
+  end
 end
 
 desc "generate gemspec"
@@ -36,92 +37,9 @@ task 'travis-api.gemspec' do
    end
 
    File.open('travis-api.gemspec', 'w') { |f| f << content }
-end
-
-task default: 'travis-api.gemspec'
-
-tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__)
-Dir.glob(tasks_path).each { |r| import r }
-
-module ActiveRecord
-  class Migration
-    class << self
-      attr_accessor :disable_ddl_transaction
  end
 
-    # Disable DDL transactions for this migration.
+ task default: 'travis-api.gemspec'
-    def self.disable_ddl_transaction!
-      @disable_ddl_transaction = true
-    end
 
-    def disable_ddl_transaction # :nodoc:
+ tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__)
-      self.class.disable_ddl_transaction
+ Dir.glob(tasks_path).each { |r| import r }
-    end
-  end
-
-  class Migrator
-    def use_transaction?(migration)
-      !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions?
-    end
-
-    def ddl_transaction(migration, &block)
-      if use_transaction?(migration)
-        Base.transaction { block.call }
-      else
-        block.call
-      end
-    end
-
-    def migrate(&block)
-      current = migrations.detect { |m| m.version == current_version }
-      target = migrations.detect { |m| m.version == @target_version }
-
-      if target.nil? && @target_version && @target_version > 0
-        raise UnknownMigrationVersionError.new(@target_version)
-      end
-
-      start = up? ? 0 : (migrations.index(current) || 0)
-      finish = migrations.index(target) || migrations.size - 1
-      runnable = migrations[start..finish]
-
-      # skip the last migration if we're headed down, but not ALL the way down
-      runnable.pop if down? && target
-
-      ran = []
-      runnable.each do |migration|
-        if block && !block.call(migration)
-          next
-        end
-
-        Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger
-
-        seen = migrated.include?(migration.version.to_i)
-
-        # On our way up, we skip migrating the ones we've already migrated
-        next if up? && seen
-
-        # On our way down, we skip reverting the ones we've never migrated
-        if down? && !seen
-          migration.announce 'never migrated, skipping'; migration.write
-          next
-        end
-
-        begin
-          ddl_transaction(migration) do
-            migration.migrate(@direction)
-            record_version_state_after_migrating(migration.version)
-          end
-          ran << migration
-        rescue => e
-          canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : ""
-          raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace
-        end
-      end
-      ran
-    end
-  end
-
-  class MigrationProxy
-    delegate :disable_ddl_transaction, to: :migration
-  end
-end

config/database.yml

@@ -20,4 +20,3 @@ development:
 test:
   <<: *defaults
   database: travis_test
-

lib/travis/api/app.rb

@@ -122,7 +122,6 @@ module Travis::Api
         use Travis::Api::App::Middleware::Logging
         use Travis::Api::App::Middleware::ScopeCheck
         use Travis::Api::App::Middleware::UserAgentTracker
-        use Travis::Api::App::Middleware::Metriks
 
         # make sure this is below ScopeCheck so we have the token
         use Rack::Attack if Endpoint.production?
@@ -133,6 +132,9 @@ module Travis::Api
         # rewrite should come after V3 hook
         use Travis::Api::App::Middleware::Rewrite
 
+        # v3 has its own metriks
+        use Travis::Api::App::Middleware::Metriks
+
         SettingsEndpoint.subclass :env_vars
         if Travis.config.endpoints.ssh_key
           SingletonSettingsEndpoint.subclass :ssh_key

lib/travis/api/v2/http/build.rb

@@ -17,7 +17,7 @@ module Travis
           def data
             {
               'build'  => build_data(build),
-              'commit' => commit_data(build.commit),
+              'commit' => commit_data(build.commit, build.repository),
               'jobs'   => options[:include_jobs] ? build.matrix.map { |job| job_data(job) } : [],
               'annotations' => options[:include_jobs] ? Annotations.new(annotations(build), @options).data["annotations"] : [],
             }
@@ -44,11 +44,12 @@ module Travis
               }
             end
 
-            def commit_data(commit)
+            def commit_data(commit, repository)
               {
                 'id' => commit.id,
                 'sha' => commit.commit,
                 'branch' => commit.branch,
+                'branch_is_default' => branch_is_default(commit, repository),
                 'message' => commit.message,
                 'committed_at' => format_date(commit.committed_at),
                 'author_name' => commit.author_name,
@@ -78,6 +79,10 @@ module Travis
               }
             end
 
+            def branch_is_default(commit, repository)
+              repository.default_branch == commit.branch
+            end
+
             def annotations(build)
               build.matrix.map(&:annotations).flatten
             end

lib/travis/api/v2/http/job.rb

@@ -15,7 +15,7 @@ module Travis
           def data
             {
               'job' => job_data(job),
-              'commit' => commit_data(job.commit),
+              'commit' => commit_data(job.commit, job.repository),
               'annotations' => Annotations.new(job.annotations, @options).data["annotations"],
             }
           end
@@ -42,11 +42,12 @@ module Travis
               }
             end
 
-            def commit_data(commit)
+            def commit_data(commit, repository)
               {
                 'id' => commit.id,
                 'sha' => commit.commit,
                 'branch' => commit.branch,
+                'branch_is_default' => branch_is_default(commit, repository),
                 'message' => commit.message,
                 'committed_at' => format_date(commit.committed_at),
                 'author_name' => commit.author_name,
@@ -56,6 +57,10 @@ module Travis
                 'compare_url' => commit.compare_url,
               }
             end
+
+            def branch_is_default(commit, repository)
+              repository.default_branch == commit.branch
+            end
         end
       end
     end

lib/travis/api/v3.rb

@@ -35,6 +35,7 @@ module Travis
       NotImplemented      = ServerError        .create('request not (yet) implemented', status: 501)
       RequestLimitReached = ClientError        .create('request limit reached for resource', status: 429)
       AlreadySyncing      = ClientError        .create('sync already in progress', status: 409)
+      MethodNotAllowed    = ClientError        .create('method not allowed', status: 405)
     end
   end
 end

lib/travis/api/v3/metrics.rb

@@ -0,0 +1,103 @@
+require 'metriks'
+
+module Travis::API::V3
+  class Metrics
+    class MetriksTracker
+      def initialize(prefix: "api.v3")
+        @prefx = prefix
+      end
+
+      def time(name, duration)
+        ::Metriks.timer("#{@prefix}.#{name}").update(duration)
+      end
+
+      def mark(name)
+        ::Metriks.meter("#{@prefix}.#{name}").mark
+      end
+    end
+
+    class Processor
+      attr_reader :queue, :tracker
+
+      def initialize(queue_size: 1000, tracker: MetriksTracker.new)
+        @tracker = tracker
+        @queue   = queue_size ? ::SizedQueue.new(queue_size) : ::Queue.new
+      end
+
+      def create(**options)
+        Metrics.new(self, **options)
+      end
+
+      def start
+        Thread.new { loop { process(queue.pop) } }
+      end
+
+      def process(metrics)
+        metrics.process(tracker)
+      rescue Exception => e
+        $stderr.puts e.message, e.backtrace
+      end
+    end
+
+    def initialize(processor, time: Time.now)
+      @processor  = processor
+      @start_time = time
+      @name_after = nil
+      @ticks      = []
+      @success    = nil
+      @name       = "unknown".freeze
+    end
+
+    def tick(event, time: Time.now)
+      @ticks << [event, time]
+      self
+    end
+
+    def success(**options)
+      finish(true, **options)
+    end
+
+    def failure(**options)
+      finish(false, **options)
+    end
+
+    def name_after(factory)
+      @name       = nil
+      @name_after = factory
+      self
+    end
+
+    def finish(success, time: Time.now, status: nil)
+      @success  = !!success
+      @status   = status
+      @status ||= success ? 200 : 500
+      @end_time = time
+      @processor.queue << self
+      self
+    end
+
+    def name
+      @name ||= @name_after.name[/[^:]+::[^:]+$/].underscore.tr(?/.freeze, ?..freeze)
+    end
+
+    def process(tracker)
+      tracker.mark("status.#{@status}")
+
+      if @success
+        process_ticks(tracker)
+        tracker.time("#{name}.overall", @end_time - @start_time)
+        tracker.mark("#{name}.success")
+      else
+        tracker.mark("#{name}.failure")
+      end
+    end
+
+    def process_ticks(tracker)
+      start = @start_time
+      @ticks.each do |event, time|
+        tracker.time("#{name}.#{event}", time - start)
+        start = time
+      end
+    end
+  end
+end

lib/travis/api/v3/router.rb

@@ -1,30 +1,50 @@
 module Travis::API::V3
   class Router
     include Travis::API::V3
-    attr_accessor :routes
+    attr_accessor :routes, :metrics_processor
 
     def initialize(routes = Routes)
       @routes            = routes
+      @metrics_processor = Metrics::Processor.new
+
+      metrics_processor.start
       routes.draw_routes
     end
 
     def call(env)
       return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze
+      metrics         = @metrics_processor.create
       access_control  = AccessControl.new(env)
-      factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze])
       env_params      = params(env)
+      factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze])
+
 
       raise NotFound unless factory
+      metrics.name_after(factory)
 
       filtered = factory.filter_params(env_params)
       service  = factory.new(access_control, filtered.merge(params))
+
+      metrics.tick(:prepare)
       result   = service.run
+      metrics.tick(:service)
 
       env_params.each_key { |key| result.ignored_param(key, reason: "not whitelisted".freeze) unless filtered.include?(key) }
-      render(result, env_params, env)
+      response = render(result, env_params, env)
+
+      metrics.tick(:renderer)
+      metrics.success(status: response[0])
+      response
     rescue Error => error
+      metrics.tick(:service)
+
       result   = Result.new(access_control, :error, error)
-      V3.response(result.render(env_params, env), {}, status: error.status)
+      response = V3.response(result.render(env_params, env), {}, status: error.status)
+
+      metrics.tick(:rendered)
+      metrics.failure(status: error.status)
+
+      response
     end
 
     def render(result, env_params, env)

spec/spec_helper.rb

@@ -3,6 +3,7 @@ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = ENV['ENV'] = 'test'
 require 'support/coverage'
 
 require 'rspec'
+require 'rspec/its'
 require 'database_cleaner'
 require 'sinatra/test_helpers'
 require 'logger'

spec/unit/api/v2/http/build_spec.rb

@@ -29,6 +29,7 @@ describe Travis::Api::V2::Http::Build do
       'id' => 1,
       'sha' => '62aae5f70ceee39123ef',
       'branch' => 'master',
+      'branch_is_default' => true,
       'message' => 'the commit message',
       'compare_url' => 'https://github.com/svenfuchs/minimal/compare/master...develop',
       'committed_at' => json_format_time(Time.now.utc - 1.hour),

spec/unit/api/v2/http/job_spec.rb

@@ -31,6 +31,7 @@ describe Travis::Api::V2::Http::Job do
       'sha' => '62aae5f70ceee39123ef',
       'message' => 'the commit message',
       'branch' => 'master',
+      'branch_is_default' => true,
       'message' => 'the commit message',
       'committed_at' => json_format_time(Time.now.utc - 1.hour),
       'committer_name' => 'Sven Fuchs',

spec/v3/error_handling_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Travis::API::V3::ServiceIndex do
+  let(:headers) {{  }}
+  let(:path)      { "/v3/repo/1/enable"         }
+  let(:json)      { JSON.load(response.body) }
+  let(:response)  { get(path, {}, headers)   }
+  let(:resources) { json.fetch('resources')  }
+
+  it "handles wrong HTTP method with 405 status" do
+
+    response.status.should == 405
+  end
+
+end

spec/v3/metrics_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Travis::API::V3::Metrics do
+  class TestProcessor
+    attr_reader :times, :marks, :queue
+
+    def initialize
+      @queue = []
+      @times = []
+      @marks = []
+    end
+
+    def time(*args)
+      times << args
+    end
+
+    def mark(name)
+      marks << name
+    end
+  end
+
+  subject(:processor) { TestProcessor.new                                }
+  let(:metric)        { described_class.new(processor, time: Time.at(0)) }
+
+  before do
+    metric.name_after(Travis::API::V3::Services::Branch::Find)
+    metric.tick(:example,       time: Time.at(10))
+    metric.tick(:other_example, time: Time.at(15))
+    metric.success(time: Time.at(25))
+    metric.process(processor)
+  end
+
+  its(:queue) { should be == [metric] }
+
+  its(:times) { should be == [
+    ["branch.find.example",       10.0],
+    ["branch.find.other_example", 5.0],
+    ["branch.find.overall",       25.0]
+  ] }
+
+  its(:marks) { should be == ["status.200", "branch.find.success"] }
+end