diff --git a/.buildpacks b/.buildpacks index eb20a5d0..36201cad 100644 --- a/.buildpacks +++ b/.buildpacks @@ -1,2 +1,3 @@ https://github.com/heroku/heroku-buildpack-ruby.git https://github.com/drogus/last-commit-sha-buildpack.git +https://github.com/ryandotsmith/nginx-buildpack.git diff --git a/.gitignore b/.gitignore index bfabbb0a..e51c7335 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ config/travis.yml config/database.yml -.yardoc -log/ -vendor +config/nginx.conf config/skylight.yml + +tmp/ +log/ +logs/ + +vendor + +.yardoc .coverage *.env -tmp diff --git a/Procfile b/Procfile index cfffe7c2..a9a67e8c 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: bundle exec je ./script/server +web: ./script/server console: bundle exec je ./script/console sidekiq: bundle exec je sidekiq -c 4 -r ./lib/travis/sidekiq.rb -q build_cancellations, -q build_restarts, -q job_cancellations, -q job_restarts diff --git a/bin/start-nginx b/bin/start-nginx index f49c0be4..b0d1f589 100755 --- a/bin/start-nginx +++ b/bin/start-nginx @@ -1,3 +1,82 @@ -#!/bin/sh +#!/usr/bin/env bash -"$@" +# make sure we kill all child processes once done +trap '{ pkill -P $$; rm -f config/nginx.conf; exit 255; }' EXIT + +if [ -f bin/nginx ]; then + nginx=bin/nginx +else + which nginx &>/dev/null || { echo "nginx not found" && exit 1; } + nginx=nginx +fi + +psmgr=$tmp_dir/nginx-buildpack-wait +rm -f $psmgr +mkfifo $psmgr + +#Evaluate config to get $PORT +erb config/nginx.conf.erb > config/nginx.conf + +n=1 +while getopts :f option ${@:1:2} +do + case "${option}" + in + f) FORCE=$OPTIND; n=$((n+1));; + esac +done + +#Initialize log directory. +mkdir -p logs/nginx +touch logs/nginx/access.log logs/nginx/error.log +echo 'buildpack=nginx at=logs-initialized' + +#Start log redirection. +( + #Redirect NGINX logs to stdout. + tail -qF -n 0 logs/nginx/*.log + echo 'logs' >$psmgr +) & + +#Start App Server +( + #Take the command passed to this bin and start it. + #E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb + COMMAND=${@:$n} + echo "buildpack=nginx at=start-app cmd=$COMMAND" + $COMMAND + echo 'app' >$psmgr +) & + +if [[ -z "$FORCE" ]] +then + FILE="$tmp_dir/app-initialized" + + #We block on app-initialized so that when NGINX binds to $PORT + #are app is ready for traffic. + while [[ ! -f "$FILE" ]] + do + echo 'buildpack=nginx at=app-initialization' + sleep 1 + done + echo 'buildpack=nginx at=app-initialized' +fi + +#Start NGINX +( + #We expect nginx to run in foreground. + #We also expect a socket to be at $tmp_dir/nginx.socket. + echo 'buildpack=nginx at=nginx-start' + $nginx -p . -c config/nginx.conf + echo 'nginx' >$psmgr +) & + +#This read will block the process waiting on a msg to be put into the fifo. +#If any of the processes defined above should exit, +#a msg will be put into the fifo causing the read operation +#to un-block. The process putting the msg into the fifo +#will use it's process name as a msg so that we can print the offending +#process to stdout. +read exit_process <$psmgr +echo "buildpack=nginx at=exit process=$exit_process" +exit 1 diff --git a/config/mime.types b/config/mime.types new file mode 100644 index 00000000..18f31b3e --- /dev/null +++ b/config/mime.types @@ -0,0 +1,76 @@ + +types { + text/html html htm shtml; + text/css css; + text/xml xml; + text/cache-manifest manifest appcache; + image/gif gif; + image/jpeg jpeg jpg; + application/x-javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg; + + application/java-archive jar war ear; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.ms-excel xls; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream eot; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mpeg mpeg mpg; + video/quicktime mov; + video/x-flv flv; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb new file mode 100644 index 00000000..8866fd5a --- /dev/null +++ b/config/nginx.conf.erb @@ -0,0 +1,45 @@ +daemon off; +#Heroku dynos have at least 4 cores. +worker_processes <%= ENV['NGINX_WORKERS'] || ENV['WEB_CONCURRENCY'] || 4 %>; + +events { + <% if `uname` != "Darwin\n" %>use epoll;<% end %> + accept_mutex on; + worker_connections 1024; +} + +http { + gzip on; + gzip_comp_level 2; + gzip_min_length 512; + + server_tokens off; + + log_format l2met 'measure#nginx.service=$request_time request_id=$http_x_request_id'; + access_log logs/nginx/access.log l2met; + error_log logs/nginx/error.log; + + include mime.types; + default_type application/octet-stream; + sendfile on; + + #Must read the body in 5 seconds. + client_body_timeout 5; + + upstream app_server { + server unix:<%= ENV["tmp_dir"] %>/nginx.socket fail_timeout=0; + } + + server { + listen <%= ENV["PORT"] %>; + server_name _; + keepalive_timeout 5; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app_server; + } + } +} diff --git a/config/puma-config.rb b/config/puma-config.rb index 38d7628e..1c15535d 100644 --- a/config/puma-config.rb +++ b/config/puma-config.rb @@ -2,8 +2,8 @@ root = File.expand_path('../..', __FILE__) rackup "#{root}/config.ru" -bind 'unix:///tmp/nginx.socket' - +tmp_dir = ENV.fetch("tmp_dir", "/tmp") +bind "unix://#{tmp_dir}/nginx.socket" environment ENV['RACK_ENV'] || 'development' threads 0, 16 diff --git a/config/ruby_config.sh b/config/ruby_config.sh new file mode 100644 index 00000000..3edf1db6 --- /dev/null +++ b/config/ruby_config.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export RUBY_HEAP_MIN_SLOTS=800000 +export RUBY_GC_HEAP_INIT_SLOTS=$RUBY_HEAP_MIN_SLOTS +export RUBY_GC_MALLOC_LIMIT=59000000 +export RUBY_HEAP_SLOTS_INCREMENT=10000 +export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1 +export RUBY_HEAP_FREE_MIN=100000 diff --git a/config/unicorn.rb b/config/unicorn.rb index 1e9da525..463a017c 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -1,19 +1,16 @@ # http://michaelvanrooijen.com/articles/2011/06/01-more-concurrency-on-a-single-heroku-dyno-with-the-new-celadon-cedar-stack/ -worker_processes 4 # amount of unicorn workers to spin up -timeout 30 # restarts workers that hang for 15 seconds +worker_processes Integer(ENV.fetch('WEB_CONCURRENCY')) # amount of unicorn workers to spin up +timeout 30 # restarts workers that hang for 30 seconds -listen '/tmp/nginx.socket', backlog: 1024 +tmp_dir = ENV.fetch("tmp_dir", "/tmp") +listen File.expand_path("nginx.socket", tmp_dir), backlog: 1024 require 'fileutils' -before_fork do |server,worker| - FileUtils.touch('/tmp/app-initialized') -end +before_fork do |server, worker| + # preload travis so we can have copy on write + require 'travis/api/app' -before_exec do |server| - ENV['RUBY_HEAP_MIN_SLOTS']=800000 - ENV['RUBY_GC_MALLOC_LIMIT']=59000000 - ENV['RUBY_HEAP_SLOTS_INCREMENT']=10000 - ENV['RUBY_HEAP_SLOTS_GROWTH_FACTOR']=1 - ENV['RUBY_HEAP_FREE_MIN']=100000 + # signal to nginx we're ready + FileUtils.touch("#{tmp_dir}/app-initialized") end diff --git a/lib/travis/api/app/endpoint/home.rb b/lib/travis/api/app/endpoint/home.rb index c91612cf..ecdfbf82 100644 --- a/lib/travis/api/app/endpoint/home.rb +++ b/lib/travis/api/app/endpoint/home.rb @@ -49,6 +49,18 @@ class Travis::Api::App get '/config' do { config: settings.client_config } end + + deploy_sha = File.read(".deploy-sha") if File.exist?(".deploy-sha") + sys_info = { + web_concurrency: ENV['WEB_CONCURRENCY'], + ulimit: `echo "ulimit -u" | bash`.to_i, + dyno: ENV['DYNO'], + deploy_sha: deploy_sha + } + + get '/sysinfo' do + sys_info + end end end end diff --git a/lib/travis/api/attack.rb b/lib/travis/api/attack.rb index 9d9231e1..4d91ec31 100644 --- a/lib/travis/api/attack.rb +++ b/lib/travis/api/attack.rb @@ -60,16 +60,16 @@ class Rack::Attack end ### - # Throttle: unauthenticated requests - 50 per minute + # Throttle: unauthenticated requests - 500 per minute # Scoped by: IP address - throttle('req/ip/1min', limit: 50, period: 1.minute) do |request| + throttle('req/ip/1min', limit: 500, period: 1.minute) do |request| request.ip unless request.authenticated? end ### - # Throttle: authenticated requests - 200 per minute + # Throttle: authenticated requests - 2000 per minute # Scoped by: access token - throttle('req/token/1min', limit: 200, period: 1.minute) do |request| + throttle('req/token/1min', limit: 2000, period: 1.minute) do |request| request.identifier end diff --git a/lib/travis/api/v3/models/account.rb b/lib/travis/api/v3/models/account.rb index 53f45fd0..aff1a510 100644 --- a/lib/travis/api/v3/models/account.rb +++ b/lib/travis/api/v3/models/account.rb @@ -40,4 +40,4 @@ module Travis::API::V3 alias_method :educational, :educational? alias_method :subscribed, :subscribed? end -end \ No newline at end of file +end diff --git a/lib/travis/api/v3/queries/branches.rb b/lib/travis/api/v3/queries/branches.rb index c368fa7d..a2c8e1c3 100644 --- a/lib/travis/api/v3/queries/branches.rb +++ b/lib/travis/api/v3/queries/branches.rb @@ -1,10 +1,33 @@ module Travis::API::V3 class Queries::Branches < Query - sortable_by :name, last_build: "builds.started_at".freeze - default_sort "last_build:desc" + params :exists_on_github, prefix: :branch + + sortable_by :name, + last_build: "builds.started_at".freeze, + exists_on_github: sort_condition(:exists_on_github), + default_branch: sort_condition(name: "repositories.default_branch") + + default_sort "default_branch,exists_on_github,last_build:desc" def find(repository) - sort repository.branches + sort(filter(repository.branches), repository: repository) + end + + def sort_by(collection, field, repository: nil, **options) + return super unless field == "default_branch".freeze + + if repository + options[:sql] = sort_condition(name: quote(repository.default_branch_name)) + else + collection = collection.joins(:repository) + end + + super(collection, field, **options) + end + + def filter(list) + list = list.where(exists_on_github: bool(exists_on_github)) unless exists_on_github.nil? + list end end end diff --git a/lib/travis/api/v3/query.rb b/lib/travis/api/v3/query.rb index 488018bf..e4ea4f09 100644 --- a/lib/travis/api/v3/query.rb +++ b/lib/travis/api/v3/query.rb @@ -48,6 +48,13 @@ module Travis::API::V3 mapping.each { |key, value| sort_by[key.to_s] = prefix(value) } end + def self.sort_condition(condition) + if condition.is_a? Hash + condition = condition.map { |e| e.map { |v| prefix(v) }.join(" = ".freeze) }.join(" and ".freeze) + end + "(case when #{prefix(condition)} then 1 else 2 end)" + end + def self.sortable? !sort_by.empty? end @@ -111,14 +118,14 @@ module Travis::API::V3 value.split(?,.freeze) end - def sort(collection) + def sort(collection, **options) return collection unless sort_by = params["sort_by".freeze] || self.class.default_sort and not sort_by.empty? first = true list(sort_by).each do |field_with_order| field, order = field_with_order.split(?:.freeze, 2) order ||= "asc".freeze if sort_by? field, order - collection = sort_by(collection, field, order: order, first: first) + collection = sort_by(collection, field, order: order, first: first, **options) first = false else ignored_value("sort_by".freeze, field_with_order, reason: "not a valid sort mode".freeze) @@ -132,9 +139,9 @@ module Travis::API::V3 self.class.sort_by.include?(field) end - def sort_by(collection, field, order: nil, first: false) + def sort_by(collection, field, order: nil, first: false, sql: nil, **) raise ArgumentError, 'cannot sort by that' unless sort_by?(field, order) - actual = self.class.sort_by.fetch(field) + actual = sql || self.class.sort_by.fetch(field) line = "#{actual} #{order.upcase}" if sort_join?(collection, actual) @@ -150,6 +157,14 @@ module Travis::API::V3 !collection.reflect_on_association(field.to_sym).nil? end + def sort_condition(*args) + self.class.sort_condition(*args) + end + + def quote(value) + ActiveRecord::Base.connection.quote(value) + end + def user_condition(value) case value when String then { login: value } diff --git a/lib/travis/api/v3/services/branches/find.rb b/lib/travis/api/v3/services/branches/find.rb index cd618c70..34cf0ef1 100644 --- a/lib/travis/api/v3/services/branches/find.rb +++ b/lib/travis/api/v3/services/branches/find.rb @@ -1,5 +1,6 @@ module Travis::API::V3 class Services::Branches::Find < Service + params :exists_on_github, prefix: :branch paginate def run! diff --git a/script/server b/script/server index 01320b42..3afd1c27 100755 --- a/script/server +++ b/script/server @@ -1,8 +1,26 @@ #!/usr/bin/env bash cd "$(dirname "$0")/.." -[ $PORT ] || PORT=3000 -[ $RACK_ENV ] || RACK_ENV=development -cmd="ruby -I lib -S bundle exec ruby -I lib -S unicorn config.ru -p $PORT -E $RACK_ENV -c config/unicorn.rb" -[[ $RACK_ENV == "development" ]] && exec rerun "$cmd -l 127.0.0.1:$PORT" +. config/ruby_config.sh + +[ $PORT ] || export PORT=3000 +[ $RACK_ENV ] || export RACK_ENV=development +[ $WEB_CONCURRENCY ] || export WEB_CONCURRENCY=$(script/web_concurrency) +[ $NGINX_WORKERS ] || export NGINX_WORKERS=$(script/web_concurrency --nginx) + +echo "port=$PORT rack_env=$RACK_ENV web_concurrency=$WEB_CONCURRENCY nginx_workers=$NGINX_WORKERS" >&2 + +ruby="ruby -I lib -S" +bexc="$ruby bundle exec" + +if [[ $RACK_ENV == "production" ]]; then + export tmp_dir=/tmp +else + mkdir -p tmp + export tmp_dir=./tmp +fi + +cmd="unicorn config.ru -E $RACK_ENV -c config/unicorn.rb" +[[ $RACK_ENV == "development" ]] && cmd="rerun -b -- $cmd" +cmd="bin/start-nginx $bexec je $cmd" exec $cmd diff --git a/script/web_concurrency b/script/web_concurrency new file mode 100755 index 00000000..1e7bdec1 --- /dev/null +++ b/script/web_concurrency @@ -0,0 +1,25 @@ +#!/bin/bash + +if [[ "$RACK_ENV" == "development" ]]; then + echo -n 2 + exit 0 +fi + + +if [[ "$1" == "--nginx" ]]; then + case $(ulimit -u) in + 256) echo -n 1; exit 0;; # Standard 1x, 512MB + 512) echo -n 2; exit 0;; # Standard 2x, 1GB + 16384) echo -n 4; exit 0;; # Performance M, 2.5GB + 32768) echo -n 4; exit 0;; # Performance L, 14GB + esac +else + case $(ulimit -u) in + 256) echo -n 2; exit 0;; # Standard 1x, 512MB + 512) echo -n 4; exit 0;; # Standard 2x, 1GB + 16384) echo -n 10; exit 0;; # Performance M, 2.5GB + 32768) echo -n 50; exit 0;; # Performance L, 14GB + esac +fi + +echo -n 4 diff --git a/spec/v3/services/branches/find_spec.rb b/spec/v3/services/branches/find_spec.rb index 6a38111b..3a930db5 100644 --- a/spec/v3/services/branches/find_spec.rb +++ b/spec/v3/services/branches/find_spec.rb @@ -132,6 +132,20 @@ describe Travis::API::V3::Services::Branches::Find do } end + describe "filtering by exists_on_github" do + describe "false" do + before { get("/v3/repo/#{repo.id}/branches?branch.exists_on_github=false") } + example { expect(last_response).to be_ok } + example { expect(parsed_body["branches"]).to be_empty } + end + + describe "true" do + before { get("/v3/repo/#{repo.id}/branches?branch.exists_on_github=true") } + example { expect(last_response).to be_ok } + example { expect(parsed_body["branches"]).not_to be_empty } + end + end + describe "sorting by name" do before { get("/v3/repo/#{repo.id}/branches?sort_by=name&limit=1") } example { expect(last_response).to be_ok } @@ -176,6 +190,50 @@ describe Travis::API::V3::Services::Branches::Find do } end + describe "sorting by exists_on_github" do + before { get("/v3/repo/#{repo.id}/branches?sort_by=exists_on_github&limit=1") } + example { expect(last_response).to be_ok } + example { expect(parsed_body["@pagination"]).to be == { + "limit" => 1, + "offset" => 0, + "count" => 1, + "is_first" => true, + "is_last" => true, + "next" => nil, + "prev" => nil, + "first" => { + "@href" => "/v3/repo/#{repo.id}/branches?sort_by=exists_on_github&limit=1", + "offset" => 0, + "limit" => 1 }, + "last" => { + "@href" => "/v3/repo/#{repo.id}/branches?sort_by=exists_on_github&limit=1", + "offset" => 0, + "limit" => 1 }} + } + end + + describe "sorting by default_branch" do + before { get("/v3/repo/#{repo.id}/branches?sort_by=default_branch&limit=1") } + example { expect(last_response).to be_ok } + example { expect(parsed_body["@pagination"]).to be == { + "limit" => 1, + "offset" => 0, + "count" => 1, + "is_first" => true, + "is_last" => true, + "next" => nil, + "prev" => nil, + "first" => { + "@href" => "/v3/repo/#{repo.id}/branches?sort_by=default_branch&limit=1", + "offset" => 0, + "limit" => 1 }, + "last" => { + "@href" => "/v3/repo/#{repo.id}/branches?sort_by=default_branch&limit=1", + "offset" => 0, + "limit" => 1 }} + } + end + describe "sorting by unknown sort field" do before { get("/v3/repo/#{repo.id}/branches?sort_by=name:desc,foo&limit=1") } example { expect(last_response).to be_ok }