suzanne.soy/travis-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into cd-v3

Browse Source
This commit is contained in:
carlad 2016-03-08 17:03:40 +01:00
commit c0e01c46d2
17 changed files with 273 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.travis.yml
View File

@ -18,4 +18,7 @@ services:
- redis
before_script:
- 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace'
- 'RAILS_ENV=test bundle exec rake db:create --trace'
script:
- bundle exec rspec spec

4
Gemfile
View File

@ -1,7 +1,7 @@
source 'https://rubygems.org'
gemspec
ruby '2.1.6' if ENV.key?('DYNO')
ruby '2.1.7' if ENV.key?('DYNO')
gem 's3', github: 'travis-ci/s3'
@ -37,9 +37,11 @@ gem 'customerio'
group :test do
gem 'rspec', '~> 2.13'
gem 'rspec-its'
gem 'factory_girl', '~> 2.4.0'
gem 'mocha', '~> 0.12'
gem 'database_cleaner', '~> 0.8.0'
gem 'travis-migrations', github: 'travis-ci/travis-migrations'
end
group :development do

18
Gemfile.lock
View File

@ -50,7 +50,7 @@ GIT
GIT
remote: git://github.com/travis-ci/travis-core.git
revision: 3a9f6e4c14bb1eacb93609e1864c9c547a13c1a4
revision: 96ee8c449ebe305c5c95633cea13eb88fe978abb
specs:
travis-core (0.0.1)
actionmailer (~> 3.2.19)
@ -72,6 +72,12 @@ GIT
travis-config (~> 0.1.0)
virtus (~> 1.0.0)
GIT
remote: git://github.com/travis-ci/travis-migrations.git
revision: fcf6eea3e3122a7cbb857826db835de69974c54d
specs:
travis-migrations (0.0.1)
GIT
remote: git://github.com/travis-ci/travis-sidekiqs.git
revision: 21a2fee158e25252dd78f5fa31e81b4f6583be23
@ -87,7 +93,7 @@ GIT
GIT
remote: git://github.com/travis-ci/travis-yaml.git
revision: 9ebe328e7546c696dd374a8cf773d93276f98e4f
revision: 032caed23af8ed1ed55e9204bb91316f3ada2f74
specs:
travis-yaml (0.2.0)
@ -276,6 +282,9 @@ GEM
rspec-core (2.99.2)
rspec-expectations (2.99.2)
diff-lcs (>= 1.1.3, < 2.0)
rspec-its (1.0.1)
rspec-core (>= 2.99.0.beta1)
rspec-expectations (>= 2.99.0.beta1)
rspec-mocks (2.99.2)
sidekiq (3.3.0)
celluloid (>= 0.16.0)
@ -362,6 +371,7 @@ DEPENDENCIES
rb-fsevent (~> 0.9.1)
rerun
rspec (~> 2.13)
rspec-its
s3!
sentry-raven!
simplecov
@ -372,8 +382,12 @@ DEPENDENCIES
travis-api!
travis-config (~> 0.1.0)
travis-core!
travis-migrations!
travis-sidekiqs!
travis-support!
travis-yaml!
unicorn
yard-sinatra!
BUNDLED WITH
1.10.6

10
README.md
View File

@ -7,7 +7,7 @@ This is the app running on https://api.travis-ci.org/
1. PostgreSQL 9.3 or higher
1. Redis
1. RabbitMQ
1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally.
1. Nginx *NB: If working on Ubuntu please install Nginx manually from source. [This guide](http://www.rackspace.com/knowledge_center/article/ubuntu-and-debian-installing-nginx-from-source) is helpful but make sure you install the [latest stable version](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/#stable), include the user name on your ubuntu machine when compiling (add `--user=[yourusername]` as an option when running `./configure`), and don't follow any subsequent server configuration steps. Travis-api will start and configure its own nginx server when run locally.
## Installation
@ -17,8 +17,10 @@ This is the app running on https://api.travis-ci.org/
### Database setup
1. `rake db:create db:migrate`
2. for testing 'RAILS_ENV=test bundle exec rake db:create db:migrate --trace'
NB detail for how `rake` sets up the database can be found in the `Rakefile`. In the `namespace :db` block you will see the database name for development is hardcoded to `travis-development`. If you are using a different configuration you will have to make your own adjustments.
1. `bundle exec rake db:create`
2. for testing 'RAILS_ENV=test bundle exec rake db:create --trace'
1. Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`):
```sh-session
cd ..
@ -31,7 +33,7 @@ pg_dump -t logs travis_logs_development | psql -U postgres travis_development
Repeat the database steps for `RAILS_ENV=test`.
```sh-session
RAILS_ENV=test rake db:create db:structure:load
RAILS_ENV=test bundle exec rake db:create
pushd ../travis-logs
RAILS_ENV=test rvm jruby do bundle exec rake db:migrate
psql -c "DROP TABLE IF EXISTS logs CASCADE" -U postgres travis_test

150
Rakefile
View File

@ -1,127 +1,45 @@
require 'bundler/setup'
require 'travis'
require 'travis/engine'
require 'rake'
require 'travis/migrations'
begin
ENV['SCHEMA'] = File.expand_path('../db/schema.rb', $:.detect { |p| p.include?('travis-core') })
require 'micro_migrations'
rescue LoadError
# we can't load micro migrations on production
end
require 'travis'
task default: :spec
begin
require 'rspec/core/rake_task'
RSpec::Core::RakeTask.new
task default: :spec
rescue LoadError
warn "could not load rspec"
namespace :db do
if ENV["RAILS_ENV"] == 'test'
desc 'Create and migrate the test database'
task :create do
sh 'createdb travis_test' rescue nil
sh "psql -q travis_test < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql"
end
else
desc 'Create and migrate the development database'
task :create do
sh 'createdb travis_development' rescue nil
sh "psql -q travis_development < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql"
end
end
end
desc "generate gemspec"
task 'travis-api.gemspec' do
content = File.read 'travis-api.gemspec'
content = File.read 'travis-api.gemspec'
fields = {
authors: `git shortlog -sn`.scan(/[^\d\s].*/),
email: `git shortlog -sne`.scan(/[^<]+@[^>]+/),
files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ }
}
fields = {
authors: `git shortlog -sn`.scan(/[^\d\s].*/),
email: `git shortlog -sne`.scan(/[^<]+@[^>]+/),
files: `git ls-files`.split("\n").reject { |f| f =~ /^(\.|Gemfile)/ }
}
fields.each do |field, values|
updated = " s.#{field} = ["
updated << values.map { |v| "\n %p" % v }.join(',')
updated << "\n ]"
content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated)
end
fields.each do |field, values|
updated = " s.#{field} = ["
updated << values.map { |v| "\n %p" % v }.join(',')
updated << "\n ]"
content.sub!(/ s\.#{field} = \[\n( .*\n)* \]/, updated)
end
File.open('travis-api.gemspec', 'w') { |f| f << content }
end
File.open('travis-api.gemspec', 'w') { |f| f << content }
end
task default: 'travis-api.gemspec'
task default: 'travis-api.gemspec'
tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__)
Dir.glob(tasks_path).each { |r| import r }
module ActiveRecord
class Migration
class << self
attr_accessor :disable_ddl_transaction
end
# Disable DDL transactions for this migration.
def self.disable_ddl_transaction!
@disable_ddl_transaction = true
end
def disable_ddl_transaction # :nodoc:
self.class.disable_ddl_transaction
end
end
class Migrator
def use_transaction?(migration)
!migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions?
end
def ddl_transaction(migration, &block)
if use_transaction?(migration)
Base.transaction { block.call }
else
block.call
end
end
def migrate(&block)
current = migrations.detect { |m| m.version == current_version }
target = migrations.detect { |m| m.version == @target_version }
if target.nil? && @target_version && @target_version > 0
raise UnknownMigrationVersionError.new(@target_version)
end
start = up? ? 0 : (migrations.index(current) || 0)
finish = migrations.index(target) || migrations.size - 1
runnable = migrations[start..finish]
# skip the last migration if we're headed down, but not ALL the way down
runnable.pop if down? && target
ran = []
runnable.each do |migration|
if block && !block.call(migration)
next
end
Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger
seen = migrated.include?(migration.version.to_i)
# On our way up, we skip migrating the ones we've already migrated
next if up? && seen
# On our way down, we skip reverting the ones we've never migrated
if down? && !seen
migration.announce 'never migrated, skipping'; migration.write
next
end
begin
ddl_transaction(migration) do
migration.migrate(@direction)
record_version_state_after_migrating(migration.version)
end
ran << migration
rescue => e
canceled_msg = Base.connection.supports_ddl_transactions? ? "this and " : ""
raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace
end
end
ran
end
end
class MigrationProxy
delegate :disable_ddl_transaction, to: :migration
end
end
tasks_path = File.expand_path('../lib/tasks/*.rake', __FILE__)
Dir.glob(tasks_path).each { |r| import r }

1
config/database.yml
View File

@ -20,4 +20,3 @@ development:
test:
<<: *defaults
database: travis_test

4
lib/travis/api/app.rb
View File

@ -122,7 +122,6 @@ module Travis::Api
use Travis::Api::App::Middleware::Logging
use Travis::Api::App::Middleware::ScopeCheck
use Travis::Api::App::Middleware::UserAgentTracker
use Travis::Api::App::Middleware::Metriks
# make sure this is below ScopeCheck so we have the token
use Rack::Attack if Endpoint.production?
@ -133,6 +132,9 @@ module Travis::Api
# rewrite should come after V3 hook
use Travis::Api::App::Middleware::Rewrite
# v3 has its own metriks
use Travis::Api::App::Middleware::Metriks
SettingsEndpoint.subclass :env_vars
if Travis.config.endpoints.ssh_key
SingletonSettingsEndpoint.subclass :ssh_key

9
lib/travis/api/v2/http/build.rb
View File

@ -17,7 +17,7 @@ module Travis
def data
{
'build' => build_data(build),
'commit' => commit_data(build.commit),
'commit' => commit_data(build.commit, build.repository),
'jobs' => options[:include_jobs] ? build.matrix.map { |job| job_data(job) } : [],
'annotations' => options[:include_jobs] ? Annotations.new(annotations(build), @options).data["annotations"] : [],
}
@ -44,11 +44,12 @@ module Travis
}
end
def commit_data(commit)
def commit_data(commit, repository)
{
'id' => commit.id,
'sha' => commit.commit,
'branch' => commit.branch,
'branch_is_default' => branch_is_default(commit, repository),
'message' => commit.message,
'committed_at' => format_date(commit.committed_at),
'author_name' => commit.author_name,
@ -78,6 +79,10 @@ module Travis
}
end
def branch_is_default(commit, repository)
repository.default_branch == commit.branch
end
def annotations(build)
build.matrix.map(&:annotations).flatten
end

9
lib/travis/api/v2/http/job.rb
View File

@ -15,7 +15,7 @@ module Travis
def data
{
'job' => job_data(job),
'commit' => commit_data(job.commit),
'commit' => commit_data(job.commit, job.repository),
'annotations' => Annotations.new(job.annotations, @options).data["annotations"],
}
end
@ -42,11 +42,12 @@ module Travis
}
end
def commit_data(commit)
def commit_data(commit, repository)
{
'id' => commit.id,
'sha' => commit.commit,
'branch' => commit.branch,
'branch_is_default' => branch_is_default(commit, repository),
'message' => commit.message,
'committed_at' => format_date(commit.committed_at),
'author_name' => commit.author_name,
@ -56,6 +57,10 @@ module Travis
'compare_url' => commit.compare_url,
}
end
def branch_is_default(commit, repository)
repository.default_branch == commit.branch
end
end
end
end

1
lib/travis/api/v3.rb
View File

@ -35,6 +35,7 @@ module Travis
NotImplemented = ServerError .create('request not (yet) implemented', status: 501)
RequestLimitReached = ClientError .create('request limit reached for resource', status: 429)
AlreadySyncing = ClientError .create('sync already in progress', status: 409)
MethodNotAllowed = ClientError .create('method not allowed', status: 405)
end
end
end

103
lib/travis/api/v3/metrics.rb Normal file
View File

@ -0,0 +1,103 @@
require 'metriks'
module Travis::API::V3
class Metrics
class MetriksTracker
def initialize(prefix: "api.v3")
@prefx = prefix
end
def time(name, duration)
::Metriks.timer("#{@prefix}.#{name}").update(duration)
end
def mark(name)
::Metriks.meter("#{@prefix}.#{name}").mark
end
end
class Processor
attr_reader :queue, :tracker
def initialize(queue_size: 1000, tracker: MetriksTracker.new)
@tracker = tracker
@queue = queue_size ? ::SizedQueue.new(queue_size) : ::Queue.new
end
def create(**options)
Metrics.new(self, **options)
end
def start
Thread.new { loop { process(queue.pop) } }
end
def process(metrics)
metrics.process(tracker)
rescue Exception => e
$stderr.puts e.message, e.backtrace
end
end
def initialize(processor, time: Time.now)
@processor = processor
@start_time = time
@name_after = nil
@ticks = []
@success = nil
@name = "unknown".freeze
end
def tick(event, time: Time.now)
@ticks << [event, time]
self
end
def success(**options)
finish(true, **options)
end
def failure(**options)
finish(false, **options)
end
def name_after(factory)
@name = nil
@name_after = factory
self
end
def finish(success, time: Time.now, status: nil)
@success = !!success
@status = status
@status ||= success ? 200 : 500
@end_time = time
@processor.queue << self
self
end
def name
@name ||= @name_after.name[/[^:]+::[^:]+$/].underscore.tr(?/.freeze, ?..freeze)
end
def process(tracker)
tracker.mark("status.#{@status}")
if @success
process_ticks(tracker)
tracker.time("#{name}.overall", @end_time - @start_time)
tracker.mark("#{name}.success")
else
tracker.mark("#{name}.failure")
end
end
def process_ticks(tracker)
start = @start_time
@ticks.each do |event, time|
tracker.time("#{name}.#{event}", time - start)
start = time
end
end
end
end

38
lib/travis/api/v3/router.rb
View File

@ -1,30 +1,50 @@
module Travis::API::V3
class Router
include Travis::API::V3
attr_accessor :routes
attr_accessor :routes, :metrics_processor
def initialize(routes = Routes)
@routes = routes
@routes = routes
@metrics_processor = Metrics::Processor.new
metrics_processor.start
routes.draw_routes
end
def call(env)
return service_index(env) if env['PATH_INFO'.freeze] == ?/.freeze
metrics = @metrics_processor.create
access_control = AccessControl.new(env)
factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze])
env_params = params(env)
factory, params = routes.factory_for(env['REQUEST_METHOD'.freeze], env['PATH_INFO'.freeze])
raise NotFound unless factory
metrics.name_after(factory)
filtered = factory.filter_params(env_params)
service = factory.new(access_control, filtered.merge(params))
result = service.run
filtered = factory.filter_params(env_params)
service = factory.new(access_control, filtered.merge(params))
metrics.tick(:prepare)
result = service.run
metrics.tick(:service)
env_params.each_key { |key| result.ignored_param(key, reason: "not whitelisted".freeze) unless filtered.include?(key) }
render(result, env_params, env)
response = render(result, env_params, env)
metrics.tick(:renderer)
metrics.success(status: response[0])
response
rescue Error => error
result = Result.new(access_control, :error, error)
V3.response(result.render(env_params, env), {}, status: error.status)
metrics.tick(:service)
result = Result.new(access_control, :error, error)
response = V3.response(result.render(env_params, env), {}, status: error.status)
metrics.tick(:rendered)
metrics.failure(status: error.status)
response
end
def render(result, env_params, env)

1
spec/spec_helper.rb
View File

@ -3,6 +3,7 @@ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = ENV['ENV'] = 'test'
require 'support/coverage'
require 'rspec'
require 'rspec/its'
require 'database_cleaner'
require 'sinatra/test_helpers'
require 'logger'

1
spec/unit/api/v2/http/build_spec.rb
View File

@ -29,6 +29,7 @@ describe Travis::Api::V2::Http::Build do
'id' => 1,
'sha' => '62aae5f70ceee39123ef',
'branch' => 'master',
'branch_is_default' => true,
'message' => 'the commit message',
'compare_url' => 'https://github.com/svenfuchs/minimal/compare/master...develop',
'committed_at' => json_format_time(Time.now.utc - 1.hour),

1
spec/unit/api/v2/http/job_spec.rb
View File

@ -31,6 +31,7 @@ describe Travis::Api::V2::Http::Job do
'sha' => '62aae5f70ceee39123ef',
'message' => 'the commit message',
'branch' => 'master',
'branch_is_default' => true,
'message' => 'the commit message',
'committed_at' => json_format_time(Time.now.utc - 1.hour),
'committer_name' => 'Sven Fuchs',

15
spec/v3/error_handling_spec.rb Normal file
View File

@ -0,0 +1,15 @@
require 'spec_helper'
describe Travis::API::V3::ServiceIndex do
let(:headers) {{ }}
let(:path) { "/v3/repo/1/enable" }
let(:json) { JSON.load(response.body) }
let(:response) { get(path, {}, headers) }
let(:resources) { json.fetch('resources') }
it "handles wrong HTTP method with 405 status" do
response.status.should == 405
end
end

42
spec/v3/metrics_spec.rb Normal file
View File

@ -0,0 +1,42 @@
require 'spec_helper'
describe Travis::API::V3::Metrics do
class TestProcessor
attr_reader :times, :marks, :queue
def initialize
@queue = []
@times = []
@marks = []
end
def time(*args)
times << args
end
def mark(name)
marks << name
end
end
subject(:processor) { TestProcessor.new }
let(:metric) { described_class.new(processor, time: Time.at(0)) }
before do
metric.name_after(Travis::API::V3::Services::Branch::Find)
metric.tick(:example, time: Time.at(10))
metric.tick(:other_example, time: Time.at(15))
metric.success(time: Time.at(25))
metric.process(processor)
end
its(:queue) { should be == [metric] }
its(:times) { should be == [
["branch.find.example", 10.0],
["branch.find.other_example", 5.0],
["branch.find.overall", 25.0]
] }
its(:marks) { should be == ["status.200", "branch.find.success"] }
end