diff --git a/.buildpacks b/.buildpacks index eb20a5d0..36201cad 100644 --- a/.buildpacks +++ b/.buildpacks @@ -1,2 +1,3 @@ https://github.com/heroku/heroku-buildpack-ruby.git https://github.com/drogus/last-commit-sha-buildpack.git +https://github.com/ryandotsmith/nginx-buildpack.git diff --git a/.travis.yml b/.travis.yml index 9c38f8c1..157e917a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,15 @@ language: ruby +env: + global: + - RUBY_GC_MALLOC_LIMIT=90000000 + - RUBY_FREE_MIN=200000 rvm: - - 1.9.3 + - 2.0.0 +addons: + postgresql: 9.3 before_script: - - 'RAILS_ENV=test rake db:create db:schema:load --trace' + - 'RAILS_ENV=test bundle exec rake db:create db:structure:load --trace' notifications: irc: "irc.freenode.org#travis" +services: + - redis diff --git a/Gemfile b/Gemfile index 80d6a333..58bc4852 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,4 @@ -ruby '1.9.3' rescue nil +ruby '2.0.0' source 'https://rubygems.org' gemspec @@ -9,22 +9,20 @@ gem 'travis-sidekiqs', github: 'travis-ci/travis-sidekiqs', require: nil, ref: ' gem 'sinatra' gem 'sinatra-contrib', require: nil #github: 'sinatra/sinatra-contrib', require: nil -gem 'puma', '2.3.1' +gem 'unicorn' gem 'sentry-raven', github: 'getsentry/raven-ruby' gem 'yard-sinatra', github: 'rkh/yard-sinatra' gem 'rack-contrib', github: 'rack/rack-contrib' gem 'rack-cache', '~> 1.2' gem 'rack-attack' gem 'gh' -gem 'bunny' +gem 'bunny', '~> 0.8.0' gem 'dalli' gem 'pry' gem 'metriks', '0.9.9.5' -gem 'ar-octopus', github: 'travis-ci/octopus', require: nil - group :test do - gem 'rspec', '~> 2.13.0' + gem 'rspec', '~> 2.13' gem 'factory_girl', '~> 2.4.0' gem 'mocha', '~> 0.12' gem 'database_cleaner', '~> 0.8.0' diff --git a/Gemfile.lock b/Gemfile.lock index 21b07abf..5ed03d5e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,11 +1,10 @@ GIT remote: git://github.com/getsentry/raven-ruby.git - revision: 299f326b420f9d80cfeea9d4e7bfd66ac93f5ad2 + revision: f6c79103e1ff07abc8a1863b5d2ec43d9527e0ed specs: - sentry-raven (0.5.0) + sentry-raven (0.6.0) faraday (>= 0.7.6) hashie (>= 1.1.0) - multi_json (~> 1.0) uuidtools GIT @@ -17,22 +16,14 @@ GIT GIT remote: git://github.com/rkh/yard-sinatra.git - revision: e61831bca0431b35eaa62fdd18acbc65f81322af + revision: cebeb4fc5c0ddddb8e46b0a2222900b3df9150ac specs: yard-sinatra (1.0.0) yard (~> 0.7) -GIT - remote: git://github.com/travis-ci/octopus.git - revision: 2d4cca475479516f47c3144971205f50c335ad35 - specs: - ar-octopus (0.5.0beta) - activerecord (>= 2.3.0) - activesupport (>= 2.3.0) - GIT remote: git://github.com/travis-ci/travis-core.git - revision: d57da369012458ccd513a5a91f20a807e50d41fd + revision: c251a6e0fa1c98723fa877a0bc4ee1d27190a12c specs: travis-core (0.0.1) actionmailer (~> 3.2.12) @@ -48,6 +39,7 @@ GIT rake redis (~> 3.0) rollout (~> 1.1.0) + s3 (~> 0.3) simple_states (~> 1.0.0) thor (~> 0.14.6) @@ -61,7 +53,7 @@ GIT GIT remote: git://github.com/travis-ci/travis-support.git - revision: c0add49c71ff3c788b4b7e5b63c11d3896d9847d + revision: d2cce5dbbee64fb1574386b16da3768feadb372a specs: travis-support (0.0.1) @@ -76,7 +68,8 @@ PATH specs: travis-api (0.0.1) backports (~> 2.5) - newrelic_rpm (~> 3.6.1.88) + memcachier + newrelic_rpm (~> 3.6.6) pg (~> 0.13.2) rack-contrib (~> 1.1) rack-ssl (~> 1.3, >= 1.3.3) @@ -90,12 +83,12 @@ PATH GEM remote: https://rubygems.org/ specs: - actionmailer (3.2.13) - actionpack (= 3.2.13) - mail (~> 2.5.3) - actionpack (3.2.13) - activemodel (= 3.2.13) - activesupport (= 3.2.13) + actionmailer (3.2.16) + actionpack (= 3.2.16) + mail (~> 2.5.4) + actionpack (3.2.16) + activemodel (= 3.2.16) + activesupport (= 3.2.16) builder (~> 3.0.0) erubis (~> 2.7.0) journey (~> 1.0.4) @@ -103,20 +96,20 @@ GEM rack-cache (~> 1.2) rack-test (~> 0.6.1) sprockets (~> 2.2.1) - activemodel (3.2.13) - activesupport (= 3.2.13) + activemodel (3.2.16) + activesupport (= 3.2.16) builder (~> 3.0.0) - activerecord (3.2.13) - activemodel (= 3.2.13) - activesupport (= 3.2.13) + activerecord (3.2.16) + activemodel (= 3.2.16) + activesupport (= 3.2.16) arel (~> 3.0.2) tzinfo (~> 0.3.29) - activesupport (3.2.13) - i18n (= 0.6.1) + activesupport (3.2.16) + i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) addressable (2.3.5) - arel (3.0.2) - atomic (1.1.10) + arel (3.0.3) + atomic (1.1.14) avl_tree (1.1.3) backports (2.8.2) builder (3.0.4) @@ -125,7 +118,7 @@ GEM facter (>= 1.6.12) timers (>= 1.0.0) coder (0.4.0) - coderay (1.0.9) + coderay (1.1.0) connection_pool (0.9.3) daemons (1.1.9) dalli (2.6.4) @@ -133,20 +126,20 @@ GEM activerecord rake database_cleaner (0.8.0) - diff-lcs (1.2.4) - dotenv (0.8.0) + diff-lcs (1.2.5) + dotenv (0.9.0) erubis (2.7.0) eventmachine (1.0.3) - facter (1.7.1) + facter (1.7.3) factory_girl (2.4.2) activesupport - faraday (0.8.7) - multipart-post (~> 1.1) - ffi (1.9.0) + faraday (0.9.0) + multipart-post (>= 1.2, < 3) + ffi (1.9.3) foreman (0.63.0) dotenv (>= 0.7) thor (>= 0.13.6) - gh (0.11.3) + gh (0.13.0) addressable backports faraday (~> 0.8) @@ -157,82 +150,86 @@ GEM hashr (0.0.22) hike (1.2.3) hitimes (1.2.1) - i18n (0.6.1) + i18n (0.6.9) journey (1.0.4) - json (1.8.0) - listen (1.2.2) + json (1.8.1) + kgio (2.8.0) + listen (1.0.3) rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9) rb-kqueue (>= 0.2) mail (2.5.4) mime-types (~> 1.16) treetop (~> 1.4.8) + memcachier (0.0.2) metaclass (0.0.1) - method_source (0.8.1) + method_source (0.8.2) metriks (0.9.9.5) atomic (~> 1.0) avl_tree (~> 1.1.2) hitimes (~> 1.1) - mime-types (1.23) + mime-types (1.25.1) mocha (0.14.0) metaclass (~> 0.0.1) - multi_json (1.7.7) - multipart-post (1.2.0) - net-http-persistent (2.8) + multi_json (1.8.4) + multipart-post (2.0.0) + net-http-persistent (2.9) net-http-pipeline (1.0.1) - newrelic_rpm (3.6.1.88) + newrelic_rpm (3.6.9.171) pg (0.13.2) polyglot (0.3.3) - pry (0.9.12.2) - coderay (~> 1.0.5) + proxies (0.2.1) + pry (0.9.12.4) + coderay (~> 1.0) method_source (~> 0.8) slop (~> 3.4) - puma (2.3.1) - rack (>= 1.1, < 2.0) pusher (0.11.3) multi_json (~> 1.0) signature (~> 0.1.6) rack (1.4.5) - rack-attack (2.2.0) + rack-attack (2.3.0) rack rack-cache (1.2) rack (>= 0.4) - rack-protection (1.5.0) + rack-protection (1.5.1) rack rack-ssl (1.3.3) rack rack-test (0.6.2) rack (>= 1.0) - railties (3.2.13) - actionpack (= 3.2.13) - activesupport (= 3.2.13) + railties (3.2.16) + actionpack (= 3.2.16) + activesupport (= 3.2.16) rack-ssl (~> 1.3.2) rake (>= 0.8.7) rdoc (~> 3.4) thor (>= 0.14.6, < 2.0) + raindrops (0.12.0) rake (0.9.6) rb-fsevent (0.9.3) - rb-inotify (0.9.0) + rb-inotify (0.9.2) ffi (>= 0.5.0) rb-kqueue (0.2.0) ffi (>= 0.5.0) rdoc (3.12.2) json (~> 1.4) redcarpet (2.3.0) - redis (3.0.4) - redis-namespace (1.3.0) - redis (~> 3.0.0) - rerun (0.8.1) - listen (>= 1.0.3) + redis (3.0.6) + redis-namespace (1.3.2) + redis (~> 3.0.4) + rerun (0.8.2) + listen (~> 1.0.3) rollout (1.1.0) - rspec (2.13.0) - rspec-core (~> 2.13.0) - rspec-expectations (~> 2.13.0) - rspec-mocks (~> 2.13.0) - rspec-core (2.13.1) - rspec-expectations (2.13.0) + rspec (2.14.1) + rspec-core (~> 2.14.0) + rspec-expectations (~> 2.14.0) + rspec-mocks (~> 2.14.0) + rspec-core (2.14.7) + rspec-expectations (2.14.4) diff-lcs (>= 1.1.3, < 2.0) - rspec-mocks (2.13.1) + rspec-mocks (2.14.4) + s3 (0.3.18) + proxies (~> 0.2.0) sidekiq (2.5.4) celluloid (~> 0.12.0) connection_pool (~> 0.9.2) @@ -243,43 +240,46 @@ GEM simple_states (1.0.0) activesupport hashr (~> 0.0.10) - sinatra (1.4.3) + sinatra (1.4.4) rack (~> 1.4) rack-protection (~> 1.4) tilt (~> 1.3, >= 1.3.4) - sinatra-contrib (1.4.0) + sinatra-contrib (1.4.1) backports (>= 2.0) - eventmachine + multi_json rack-protection rack-test - sinatra (~> 1.4.2) + sinatra (~> 1.4.0) tilt (~> 1.3) - slop (3.4.5) + slop (3.4.7) sprockets (2.2.2) hike (~> 1.2) multi_json (~> 1.0) rack (~> 1.0) tilt (~> 1.1, != 1.3.0) - thin (1.5.1) + thin (1.6.1) daemons (>= 1.0.9) - eventmachine (>= 0.12.6) + eventmachine (>= 1.0.0) rack (>= 1.0.0) thor (0.14.6) tilt (1.4.1) timers (1.1.0) - treetop (1.4.14) + treetop (1.4.15) polyglot polyglot (>= 0.3.1) - tzinfo (0.3.37) + tzinfo (0.3.38) + unicorn (4.7.0) + kgio (~> 2.6) + rack + raindrops (~> 0.7) uuidtools (2.1.4) - yard (0.8.6.2) + yard (0.8.7.3) PLATFORMS ruby DEPENDENCIES - ar-octopus! - bunny + bunny (~> 0.8.0) dalli database_cleaner (~> 0.8.0) factory_girl (~> 2.4.0) @@ -289,14 +289,13 @@ DEPENDENCIES micro_migrations! mocha (~> 0.12) pry - puma (= 2.3.1) rack-attack rack-cache (~> 1.2) rack-contrib! rake (~> 0.9.2) rb-fsevent (~> 0.9.1) rerun - rspec (~> 2.13.0) + rspec (~> 2.13) sentry-raven! sinatra sinatra-contrib @@ -304,4 +303,5 @@ DEPENDENCIES travis-core! travis-sidekiqs! travis-support! + unicorn yard-sinatra! diff --git a/Procfile b/Procfile index bbb53736..42675c0c 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: bundle exec ./script/server +web: bin/start-nginx bundle exec ./script/server console: bundle exec ./script/console diff --git a/README.md b/README.md index 5f7ac73c..94ae687d 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ Setup: Run tests: - $ RAILS_ENV=test rake db:create db:schema:load + $ RAILS_ENV=test rake db:create db:structure:load $ rake spec Run the server: - $ rake db:create db:schema:load + $ rake db:create db:structure:load $ foreman start ## Contributing diff --git a/Rakefile b/Rakefile index 29502405..3fe4f809 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,5 @@ require 'bundler/setup' -ENV['SCHEMA'] = "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/schema.rb" +ENV['DB_STRUCTURE'] = "#{Gem.loaded_specs['travis-core'].full_gem_path}/db/structure.sql" begin require 'micro_migrations' diff --git a/bin/start-nginx b/bin/start-nginx new file mode 100755 index 00000000..f49c0be4 --- /dev/null +++ b/bin/start-nginx @@ -0,0 +1,3 @@ +#!/bin/sh + +"$@" diff --git a/config.ru b/config.ru index 8fb5947a..e0be66b1 100644 --- a/config.ru +++ b/config.ru @@ -9,10 +9,26 @@ require 'core_ext/module/load_constants' models = Travis::Model.constants.map(&:to_s) only = [/^(ActiveRecord|ActiveModel|Travis|GH|#{models.join('|')})/] -skip = ['Travis::Memory', 'GH::ResponseWrapper', 'Travis::NewRelic'] +skip = ['Travis::Memory', 'GH::ResponseWrapper', 'Travis::NewRelic', 'Travis::Helpers::Legacy', 'GH::FaradayAdapter::EMSynchrony'] [Travis::Api, Travis, GH].each do |target| target.load_constants! :only => only, :skip => skip, :debug => false end +# https://help.heroku.com/tickets/92756 +class RackTimer + def initialize(app) + @app = app + end + + def call(env) + start_request = Time.now + status, headers, body = @app.call(env) + elapsed = (Time.now - start_request) * 1000 + $stdout.puts("request-id=#{env['HTTP_HEROKU_REQUEST_ID']} measure.rack-request=#{elapsed.round}ms") + [status, headers, body] + end +end + +use RackTimer run Travis::Api::App.new diff --git a/config/database.yml b/config/database.yml index 27d6ca0a..26b09327 100644 --- a/config/database.yml +++ b/config/database.yml @@ -7,6 +7,7 @@ defaults: &defaults pool: 5 min_messages: warning username: <%= ENV['USER'] %> + host: localhost production: <<: *defaults diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb new file mode 100644 index 00000000..2e0a0142 --- /dev/null +++ b/config/nginx.conf.erb @@ -0,0 +1,64 @@ +daemon off; +#Heroku dynos have 4 cores. +worker_processes 4; + +events { + use epoll; + accept_mutex on; + worker_connections 1024; +} + +http { + gzip on; + gzip_comp_level 2; + gzip_min_length 512; + + log_format l2met 'measure.nginx.service=$request_time request_id=$http_heroku_request_id'; + access_log logs/nginx/access.log l2met; + error_log logs/nginx/error.log; + + include mime.types; + default_type application/octet-stream; + sendfile on; + + #Must read the body in 5 seconds. + client_body_timeout 5; + + upstream app_server { + server unix:/tmp/nginx.socket fail_timeout=0; + } + + server { + listen <%= ENV["PORT"] %>; + server_name _; + keepalive_timeout 5; + + location / { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Expose-Headers' 'Content-Type, Cache-Control, Expires, Etag, Last-Modified'; + + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Expose-Headers' 'Content-Type, Cache-Control, Expires, Etag, Last-Modified'; + + # Tell browser to cache this pre-flight info for 20 days + add_header 'Access-Control-Max-Age' 1728000; + + add_header 'Access-Control-Allow-Methods' 'HEAD, GET, POST, PATCH, PUT, DELETE, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since'; + + add_header 'Content-Length' 0; + add_header 'Content-Type' 'text/plain charset=UTF-8'; + + return 204; + } + + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app_server; + } + } +} diff --git a/config/puma-config.rb b/config/puma-config.rb new file mode 100644 index 00000000..38d7628e --- /dev/null +++ b/config/puma-config.rb @@ -0,0 +1,9 @@ +root = File.expand_path('../..', __FILE__) + +rackup "#{root}/config.ru" + +bind 'unix:///tmp/nginx.socket' + +environment ENV['RACK_ENV'] || 'development' + +threads 0, 16 diff --git a/config/unicorn.rb b/config/unicorn.rb index 45bae390..1e9da525 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -2,3 +2,18 @@ worker_processes 4 # amount of unicorn workers to spin up timeout 30 # restarts workers that hang for 15 seconds + +listen '/tmp/nginx.socket', backlog: 1024 + +require 'fileutils' +before_fork do |server,worker| + FileUtils.touch('/tmp/app-initialized') +end + +before_exec do |server| + ENV['RUBY_HEAP_MIN_SLOTS']=800000 + ENV['RUBY_GC_MALLOC_LIMIT']=59000000 + ENV['RUBY_HEAP_SLOTS_INCREMENT']=10000 + ENV['RUBY_HEAP_SLOTS_GROWTH_FACTOR']=1 + ENV['RUBY_HEAP_FREE_MIN']=100000 +end diff --git a/docs/00_overview.md b/docs/00_overview.md index 1a134181..42fff0df 100644 --- a/docs/00_overview.md +++ b/docs/00_overview.md @@ -1,9 +1,9 @@ # Overview -**This documentation is for the v2 API. However, this endpoint does also serve the [v1 API](http://about.travis-ci.org/docs/dev/api/).** +**This documentation is for the v2 API. However, this endpoint does also serve the v1 API (undocumented).** -Welcome to the Travis API documentation. This is the API used by the official -[Travis CI](https://next.travis-ci.org) web interface, so everything the web +Welcome to the Travis CI API documentation. This is the API used by the official +[Travis CI](https://travis-ci.org) web interface, so everything the web interface is able to do is also accomplishable via the API. ## Media Types @@ -12,4 +12,15 @@ The API is currently [JSON](http://en.wikipedia.org/wiki/JSON) only. ## Clients and Libraries -* **[Travis Web](https://github.com/travis-ci/travis-web)**: The official Travis CI client, using [Ember.js](http://emberjs.com/). \ No newline at end of file +Official, maintained by the Travis CI team: + +* **[Travis Web](https://github.com/travis-ci/travis-web)**: Web interface and JavaScript library, using [Ember.js](http://emberjs.com/) +* **[travis](https://github.com/travis-ci/travis)**: Command line client and Ruby library + +Unofficial: + +* **[PHP Travis Client](https://github.com/l3l0/php-travis-client)**: PHP client library +* **[Travis Node.js](https://github.com/pwmckenna/node-travis-ci)**: Node.js client library +* **[travis-api-wrapper](https://github.com/cmaujean/travis-api-wrapper)**: Asynchronous Node.js wrapper +* **[travis-ci](https://github.com/mmalecki/node-travis-ci)**: Thin Node.js wrapper +* **[TravisMiner](https://github.com/smcintosh/travisminer)**: Ruby library for mining the Travis API diff --git a/lib/tasks/encyrpt_all_data.rake b/lib/tasks/encyrpt_all_data.rake index 02b05813..dcaf844e 100644 --- a/lib/tasks/encyrpt_all_data.rake +++ b/lib/tasks/encyrpt_all_data.rake @@ -4,7 +4,6 @@ namespace :db do Travis::Database.connect to_encrypt = { - Request => [:token], SslKey => [:private_key], Token => [:token], User => [:github_oauth_token] @@ -12,9 +11,9 @@ namespace :db do encrypted_column = Travis::Model::EncryptedColumn.new to_encrypt.each do |model, column_names| - model.find_in_batches(batch_size: 10000) do |records| + model.find_in_batches(batch_size: 500) do |records| ActiveRecord::Base.transaction do - puts "Encrypted 10000 of #{model} (last_id: #{records.last.id})" + puts "Encrypted 500 of #{model} (last_id: #{records.last.id})" records.each do |record| column_names.each do |column| @@ -25,6 +24,8 @@ namespace :db do end end end + + sleep 10 end end end diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb index fb4ac081..c15a6146 100644 --- a/lib/travis/api/app.rb +++ b/lib/travis/api/app.rb @@ -1,8 +1,13 @@ require 'travis' +require 'travis/model' +require 'travis/support/amqp' +require 'travis/states_cache' require 'backports' require 'rack' require 'rack/protection' require 'rack/contrib' +require 'dalli' +require 'memcachier' require 'rack/cache' require 'rack/attack' require 'active_record' @@ -12,6 +17,7 @@ require 'raven' require 'sidekiq' require 'metriks/reporter/logger' require 'travis/support/log_subscriber/active_record_metrics' +require 'fileutils' # Rack class implementing the HTTP API. # Instances respond to #call. @@ -32,6 +38,8 @@ module Travis::Api Rack.autoload :SSL, 'rack/ssl' + ERROR_RESPONSE = JSON.generate(error: 'Travis encountered an error, sorry :(') + # Used to track if setup already ran. def self.setup? @setup ||= false @@ -45,6 +53,7 @@ module Travis::Api def self.setup(options = {}) setup! unless setup? Endpoint.set(options) if options + FileUtils.touch('/tmp/app-initialized') end def self.new(options = {}) @@ -64,6 +73,8 @@ module Travis::Api def initialize @app = Rack::Builder.app do + use(Rack::Config) { |env| env['metriks.request.start'] ||= Time.now.utc } + Rack::Utils::HTTP_STATUS_CODES[420] = "Enhance Your Calm" use Rack::Attack Rack::Attack.blacklist('block client requesting ruby builds') do |req| @@ -74,19 +85,19 @@ module Travis::Api [ 420, {}, ['Enhance Your Calm']] end - use Travis::Api::App::Cors + use Travis::Api::App::Cors if Travis.env == 'development' use Raven::Rack if Endpoint.production? use Rack::Protection::PathTraversal use Rack::SSL if Endpoint.production? use ActiveRecord::ConnectionAdapters::ConnectionManagement use ActiveRecord::QueryCache - memcache_servers = ENV['MEMCACHE_SERVERS'] - if Travis::Features.feature_active?(:use_rack_cache) && memcache_server + memcache_servers = ENV['MEMCACHIER_SERVERS'] + if Travis::Features.feature_active?(:use_rack_cache) && memcache_servers use Rack::Cache, verbose: true, - metastore: "memcached://#{memcache_servers}/#{Travis::Api::App.deploy_sha}", - entitystore: "memcached://#{memcache_servers}/#{Travis::Api::App.deploy_sha}" + metastore: "memcached://#{memcache_servers}/meta-#{Travis::Api::App.deploy_sha}", + entitystore: "memcached://#{memcache_servers}/body-#{Travis::Api::App.deploy_sha}" end use Rack::Deflater @@ -111,7 +122,7 @@ module Travis::Api app.call(env) rescue if Endpoint.production? - [500, {'Content-Type' => 'text/plain'}, ['Travis encountered an error, sorry :(']] + [500, {'Content-Type' => 'application/json'}, [ERROR_RESPONSE]] else raise end @@ -135,8 +146,6 @@ module Travis::Api setup_database_connections - Travis::Features.start - if Travis.env == 'production' || Travis.env == 'staging' Sidekiq.configure_client do |config| config.redis = Travis.config.redis.merge(size: 1, namespace: Travis.config.sidekiq.namespace) @@ -156,25 +165,9 @@ module Travis::Api def self.setup_database_connections Travis::Database.connect - return unless Travis.config.use_database_follower? - require 'octopus' - - if Travis.env == 'production' || Travis.env == 'staging' - puts "Setting up the DB follower as a read slave" - - # Octopus checks for Rails.env, just hardcode enabled? - Octopus.instance_eval do - def enabled? - true - end - end - - ActiveRecord::Base.custom_octopus_connection = false - - ::Octopus.setup do |config| - config.shards = { :follower => Travis.config.database_follower } - config.environments = ['production', 'staging'] - end + if Travis.config.logs_database + Log.establish_connection 'logs_database' + Log::Part.establish_connection 'logs_database' end end diff --git a/lib/travis/api/app/base.rb b/lib/travis/api/app/base.rb index 4fbf40c8..082be3ea 100644 --- a/lib/travis/api/app/base.rb +++ b/lib/travis/api/app/base.rb @@ -9,6 +9,8 @@ class Travis::Api::App configure :production do require 'newrelic_rpm' + ::NewRelic::Agent.manual_start() + ::NewRelic::Agent.after_fork(:force_reconnect => true) end error NotImplementedError do @@ -17,6 +19,11 @@ class Travis::Api::App "This feature has not yet been implemented. Sorry :(\n\nPull Requests welcome!" end + error JSON::ParserError do + status 400 + "Invalid JSON in request body" + end + # hotfix?? def route_missing @app ? forward : halt(404) diff --git a/lib/travis/api/app/endpoint.rb b/lib/travis/api/app/endpoint.rb index 11c82afc..47799354 100644 --- a/lib/travis/api/app/endpoint.rb +++ b/lib/travis/api/app/endpoint.rb @@ -14,7 +14,7 @@ class Travis::Api::App # TODO hmmm? before { flash.clear } - before { content_type :json } + after { content_type :json unless content_type } error(ActiveRecord::RecordNotFound, Sinatra::NotFound) { not_found } not_found { content_type =~ /json/ ? { 'file' => 'not found' } : 'file not found' } diff --git a/lib/travis/api/app/endpoint/artifacts.rb b/lib/travis/api/app/endpoint/artifacts.rb deleted file mode 100644 index cd35a570..00000000 --- a/lib/travis/api/app/endpoint/artifacts.rb +++ /dev/null @@ -1,16 +0,0 @@ -require 'travis/api/app' - -class Travis::Api::App - class Endpoint - # Artifacts are generated by builds. Currently we only expose logs as - # artifacts - # - # **DEPRECATED** will be removed as soon as the client uses /logs/:id - class Artifacts < Endpoint - # Fetches an artifact by it's *id*. - get '/:id' do |id| - respond_with service(:find_log, params) - end - end - end -end diff --git a/lib/travis/api/app/endpoint/authorization.rb b/lib/travis/api/app/endpoint/authorization.rb index 61c34e9e..ce48fd49 100644 --- a/lib/travis/api/app/endpoint/authorization.rb +++ b/lib/travis/api/app/endpoint/authorization.rb @@ -78,6 +78,10 @@ class Travis::Api::App # # * **github_token**: GitHub token for checking authorization (required) post '/github' do + unless params[:github_token] + halt 422, { "error" => "Must pass 'github_token' parameter" } + end + { 'access_token' => github_to_travis(params[:github_token], app_id: 1, drop_token: true) } end @@ -196,6 +200,16 @@ class Travis::Api::App end class UserManager < Struct.new(:data, :token, :drop_token) + include User::Renaming + + attr_accessor :user + + def initialize(*) + super + + @user = ::User.find_by_github_id(data['id']) + end + def info(attributes = {}) info = data.to_hash.slice('name', 'login', 'gravatar_id') info.merge! attributes.stringify_keys @@ -203,26 +217,49 @@ class Travis::Api::App info end - def fetch - user = ::User.find_by_github_id(data['id']) - info = drop_token ? info : info(github_oauth_token: token) + def user_exists? + user + end - if user - user.update_attributes info - else - user = ::User.create! info + def fetch + retried ||= false + info = drop_token ? self.info : self.info(github_oauth_token: token) + + ActiveRecord::Base.transaction do + if user + rename_repos_owner(user.login, info['login']) + user.update_attributes info + else + self.user = ::User.create! info + end + + nullify_logins(user.github_id, user.login) end user + rescue ActiveRecord::RecordNotUnique + unless retried + retried = true + retry + end end end def user_for_github_token(token, drop_token = false) - data = GH.with(token: token.to_s, client_id: nil) { GH['user'] } - scopes = parse_scopes data.headers['x-oauth-scopes'] - halt 403, 'insufficient access: %p' unless acceptable? scopes + data = GH.with(token: token.to_s, client_id: nil) { GH['user'] } + scopes = parse_scopes data.headers['x-oauth-scopes'] + manager = UserManager.new(data, token, drop_token) - user = UserManager.new(data, token, drop_token).fetch + unless acceptable? scopes + # TODO: we should probably only redirect if this is a web + # oauth request, are there any other possibilities to + # consider? + url = Travis.config.oauth2.insufficient_access_redirect_url + url += "#existing-user" if manager.user_exists? + redirect to(url) + end + + user = manager.fetch halt 403, 'not a Travis user' if user.nil? user end diff --git a/lib/travis/api/app/endpoint/builds.rb b/lib/travis/api/app/endpoint/builds.rb index f956c7ad..2f1e5837 100644 --- a/lib/travis/api/app/endpoint/builds.rb +++ b/lib/travis/api/app/endpoint/builds.rb @@ -4,14 +4,50 @@ class Travis::Api::App class Endpoint class Builds < Endpoint get '/' do - name = params[:branches] ? :find_branches : :find_builds - params['ids'] = params['ids'].split(',') if params['ids'].respond_to?(:split) - respond_with service(name, params) + prefer_follower do + name = params[:branches] ? :find_branches : :find_builds + params['ids'] = params['ids'].split(',') if params['ids'].respond_to?(:split) + respond_with service(name, params) + end end get '/:id' do respond_with service(:find_build, params) end + + post '/:id/cancel' do + Metriks.meter("api.request.cancel_build").mark + + service = self.service(:cancel_build, params.merge(source: 'api')) + if !service.authorized? + json = { error: { + message: "You don't have access to cancel build(#{params[:id]})" + } } + + Metriks.meter("api.request.cancel_build.unauthorized").mark + status 403 + respond_with json + elsif !service.can_cancel? + json = { error: { + message: "The build(#{params[:id]}) can't be canceled", + code: 'cant_cancel' + } } + + Metriks.meter("api.request.cancel_build.cant_cancel").mark + status 422 + respond_with json + else + service.run + + Metriks.meter("api.request.cancel_build.success").mark + status 204 + end + end + + post '/:id/restart' do + Metriks.meter("api.request.restart_build").mark + respond_with service(:reset_model, build_id: params[:id]) + end end end end diff --git a/lib/travis/api/app/endpoint/documentation.rb b/lib/travis/api/app/endpoint/documentation.rb index 7d054b1c..b9ec5489 100644 --- a/lib/travis/api/app/endpoint/documentation.rb +++ b/lib/travis/api/app/endpoint/documentation.rb @@ -84,39 +84,29 @@ __END__ - Travis API documentation - - - - - - - - + Travis CI API documentation + + + - + - -
+

The Travis CI API

+ <% general_docs.each do |doc| %> <%= erb :entry, locals: doc %> <% end %> @@ -163,22 +153,33 @@ __END__
-