From e03f79513a0e3bfc9a6071fa2e9056e5b890b43f Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Thu, 8 Oct 2015 13:57:49 +0200 Subject: [PATCH 1/8] reinstate nginx, use it in development, add smare concurrency settings --- .gitignore | 13 +++-- bin/start-nginx | 90 ++++++++++++++++++++++++++++- config/mime.types | 76 ++++++++++++++++++++++++ config/nginx.conf.erb | 45 +++++++++++++++ config/puma-config.rb | 3 +- config/ruby_config.sh | 7 +++ config/unicorn.rb | 20 +++---- lib/travis/api/v3/models/account.rb | 2 +- script/server | 19 ++++-- script/web_concurrency | 33 +++++++++++ 10 files changed, 283 insertions(+), 25 deletions(-) create mode 100644 config/mime.types create mode 100644 config/nginx.conf.erb create mode 100644 config/ruby_config.sh create mode 100755 script/web_concurrency diff --git a/.gitignore b/.gitignore index bfabbb0a..e51c7335 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,14 @@ config/travis.yml config/database.yml -.yardoc -log/ -vendor +config/nginx.conf config/skylight.yml + +tmp/ +log/ +logs/ + +vendor + +.yardoc .coverage *.env -tmp diff --git a/bin/start-nginx b/bin/start-nginx index f49c0be4..1d0e3db4 100755 --- a/bin/start-nginx +++ b/bin/start-nginx @@ -1,3 +1,89 @@ -#!/bin/sh +#!/usr/bin/env bash -"$@" +# make sure we kill all child processes once done +trap '{ pkill -P $$; rm -f config/nginx.conf; exit 255; }' EXIT + +if [[ $RACK_ENV == "production" ]]; then + export tmp_dir=/tmp +else + mkdir -p tmp + export tmp_dir=./tmp +fi + +if [ -f bin/nginx ]; then + nginx=bin/nginx +else + which nginx &>/dev/null || { echo "nginx not found" && exit 1; } + nginx=nginx +fi + +psmgr=$tmp_dir/nginx-buildpack-wait +rm -f $psmgr +mkfifo $psmgr + +#Evaluate config to get $PORT +erb config/nginx.conf.erb > config/nginx.conf + +n=1 +while getopts :f option ${@:1:2} +do + case "${option}" + in + f) FORCE=$OPTIND; n=$((n+1));; + esac +done + +#Initialize log directory. +mkdir -p logs/nginx +touch logs/nginx/access.log logs/nginx/error.log +echo 'buildpack=nginx at=logs-initialized' + +#Start log redirection. +( + #Redirect NGINX logs to stdout. + tail -qF -n 0 logs/nginx/*.log + echo 'logs' >$psmgr +) & + +#Start App Server +( + #Take the command passed to this bin and start it. + #E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb + COMMAND=${@:$n} + echo "buildpack=nginx at=start-app cmd=$COMMAND" + $COMMAND + echo 'app' >$psmgr +) & + +if [[ -z "$FORCE" ]] +then + FILE="$tmp_dir/app-initialized" + + #We block on app-initialized so that when NGINX binds to $PORT + #are app is ready for traffic. + while [[ ! -f "$FILE" ]] + do + echo 'buildpack=nginx at=app-initialization' + sleep 1 + done + echo 'buildpack=nginx at=app-initialized' +fi + +#Start NGINX +( + #We expect nginx to run in foreground. + #We also expect a socket to be at $tmp_dir/nginx.socket. + echo 'buildpack=nginx at=nginx-start' + $nginx -p . -c config/nginx.conf + echo 'nginx' >$psmgr +) & + +#This read will block the process waiting on a msg to be put into the fifo. +#If any of the processes defined above should exit, +#a msg will be put into the fifo causing the read operation +#to un-block. The process putting the msg into the fifo +#will use it's process name as a msg so that we can print the offending +#process to stdout. +read exit_process <$psmgr +echo "buildpack=nginx at=exit process=$exit_process" +exit 1 diff --git a/config/mime.types b/config/mime.types new file mode 100644 index 00000000..18f31b3e --- /dev/null +++ b/config/mime.types @@ -0,0 +1,76 @@ + +types { + text/html html htm shtml; + text/css css; + text/xml xml; + text/cache-manifest manifest appcache; + image/gif gif; + image/jpeg jpeg jpg; + application/x-javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg; + + application/java-archive jar war ear; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.ms-excel xls; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream eot; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mpeg mpeg mpg; + video/quicktime mov; + video/x-flv flv; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} diff --git a/config/nginx.conf.erb b/config/nginx.conf.erb new file mode 100644 index 00000000..8866fd5a --- /dev/null +++ b/config/nginx.conf.erb @@ -0,0 +1,45 @@ +daemon off; +#Heroku dynos have at least 4 cores. +worker_processes <%= ENV['NGINX_WORKERS'] || ENV['WEB_CONCURRENCY'] || 4 %>; + +events { + <% if `uname` != "Darwin\n" %>use epoll;<% end %> + accept_mutex on; + worker_connections 1024; +} + +http { + gzip on; + gzip_comp_level 2; + gzip_min_length 512; + + server_tokens off; + + log_format l2met 'measure#nginx.service=$request_time request_id=$http_x_request_id'; + access_log logs/nginx/access.log l2met; + error_log logs/nginx/error.log; + + include mime.types; + default_type application/octet-stream; + sendfile on; + + #Must read the body in 5 seconds. + client_body_timeout 5; + + upstream app_server { + server unix:<%= ENV["tmp_dir"] %>/nginx.socket fail_timeout=0; + } + + server { + listen <%= ENV["PORT"] %>; + server_name _; + keepalive_timeout 5; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_pass http://app_server; + } + } +} diff --git a/config/puma-config.rb b/config/puma-config.rb index 38d7628e..45527340 100644 --- a/config/puma-config.rb +++ b/config/puma-config.rb @@ -2,8 +2,7 @@ root = File.expand_path('../..', __FILE__) rackup "#{root}/config.ru" -bind 'unix:///tmp/nginx.socket' - +bind "unix://#{ENV["tmp_dir"]}/nginx.socket" environment ENV['RACK_ENV'] || 'development' threads 0, 16 diff --git a/config/ruby_config.sh b/config/ruby_config.sh new file mode 100644 index 00000000..3edf1db6 --- /dev/null +++ b/config/ruby_config.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export RUBY_HEAP_MIN_SLOTS=800000 +export RUBY_GC_HEAP_INIT_SLOTS=$RUBY_HEAP_MIN_SLOTS +export RUBY_GC_MALLOC_LIMIT=59000000 +export RUBY_HEAP_SLOTS_INCREMENT=10000 +export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1 +export RUBY_HEAP_FREE_MIN=100000 diff --git a/config/unicorn.rb b/config/unicorn.rb index 1e9da525..f71a3546 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -1,19 +1,15 @@ # http://michaelvanrooijen.com/articles/2011/06/01-more-concurrency-on-a-single-heroku-dyno-with-the-new-celadon-cedar-stack/ -worker_processes 4 # amount of unicorn workers to spin up -timeout 30 # restarts workers that hang for 15 seconds +worker_processes Integer(ENV.fetch('WEB_CONCURRENCY')) # amount of unicorn workers to spin up +timeout 30 # restarts workers that hang for 30 seconds -listen '/tmp/nginx.socket', backlog: 1024 +listen File.expand_path("nginx.socket", ENV["tmp_dir"]), backlog: 1024 require 'fileutils' -before_fork do |server,worker| - FileUtils.touch('/tmp/app-initialized') -end +before_fork do |server, worker| + # preload travis so we can have copy on write + require 'travis/api/app' -before_exec do |server| - ENV['RUBY_HEAP_MIN_SLOTS']=800000 - ENV['RUBY_GC_MALLOC_LIMIT']=59000000 - ENV['RUBY_HEAP_SLOTS_INCREMENT']=10000 - ENV['RUBY_HEAP_SLOTS_GROWTH_FACTOR']=1 - ENV['RUBY_HEAP_FREE_MIN']=100000 + # signal to nginx we're ready + FileUtils.touch("#{ENV["tmp_dir"]}/app-initialized") end diff --git a/lib/travis/api/v3/models/account.rb b/lib/travis/api/v3/models/account.rb index 53f45fd0..aff1a510 100644 --- a/lib/travis/api/v3/models/account.rb +++ b/lib/travis/api/v3/models/account.rb @@ -40,4 +40,4 @@ module Travis::API::V3 alias_method :educational, :educational? alias_method :subscribed, :subscribed? end -end \ No newline at end of file +end diff --git a/script/server b/script/server index 01320b42..716823c4 100755 --- a/script/server +++ b/script/server @@ -1,8 +1,19 @@ #!/usr/bin/env bash cd "$(dirname "$0")/.." -[ $PORT ] || PORT=3000 -[ $RACK_ENV ] || RACK_ENV=development -cmd="ruby -I lib -S bundle exec ruby -I lib -S unicorn config.ru -p $PORT -E $RACK_ENV -c config/unicorn.rb" -[[ $RACK_ENV == "development" ]] && exec rerun "$cmd -l 127.0.0.1:$PORT" +. config/ruby_config.sh + +[ $PORT ] || export PORT=3000 +[ $RACK_ENV ] || export RACK_ENV=development +[ $WEB_CONCURRENCY ] || export WEB_CONCURRENCY=$(script/web_concurrency) +[ $NGINX_WORKERS ] || export NGINX_WORKERS=$(script/web_concurrency --nginx) + +echo "port=$PORT rack_env=$RACK_ENV web_concurrency=$WEB_CONCURRENCY nginx_workers=$NGINX_WORKERS" + +ruby="ruby -I lib -S" +bexc="$ruby bundle exec" + +cmd="unicorn config.ru -E $RACK_ENV -c config/unicorn.rb" +[[ $RACK_ENV == "development" ]] && cmd="rerun -b -- $cmd" +cmd="bin/start-nginx $bexec je $cmd" exec $cmd diff --git a/script/web_concurrency b/script/web_concurrency new file mode 100755 index 00000000..430b12dc --- /dev/null +++ b/script/web_concurrency @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby --disable=gems --disable=rubyopt +GC.disable + +module System + extend self + def cpu_count + return Java::Java.lang.Runtime.getRuntime.availableProcessors if defined? Java::Java + return File.read('/proc/cpuinfo').scan(/^processor\s*:/).size if File.exist? '/proc/cpuinfo' + require 'win32ole' + WIN32OLE.connect("winmgmts://").ExecQuery("select * from Win32_ComputerSystem").NumberOfProcessors + rescue LoadError + Integer `sysctl -n hw.ncpu 2>/dev/null` rescue 1 + end +end + +if ENV['DYNO'] + # we're on Heroku + case `uname -u`.to_i + when 256 then concurrency = 2 # 1x dyno + when 512 then concurrency = 4 # 2x dyno + when 32768 then concurrency = 16 # px dyno + else $stderr.puts "Unkown dyno type, selecting concurrency of 4, because" + end +end + +case ENV['RACK_ENV'] || 'development' +when 'production' then concurrency ||= System.cpu_count +when 'development' then concurrency ||= 2 # use at least two so we can be sure things work concurrencly +else concurrency ||= 1 +end + +concurrency = System.cpu_count if ARGV[0] == '--nginx' and System.cpu_count < concurrency +print concurrency From 4ed55a804efd8f432e616a04fd61722528622f0a Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 08:47:55 +0200 Subject: [PATCH 2/8] add nginx buildpack --- .buildpacks | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildpacks b/.buildpacks index eb20a5d0..36201cad 100644 --- a/.buildpacks +++ b/.buildpacks @@ -1,2 +1,3 @@ https://github.com/heroku/heroku-buildpack-ruby.git https://github.com/drogus/last-commit-sha-buildpack.git +https://github.com/ryandotsmith/nginx-buildpack.git From aadd29574c0bd537038f05698e3734fecaabc3ce Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 09:01:55 +0200 Subject: [PATCH 3/8] script/server: output to $stderr, maybe that way it will show up in heroku logs --- script/server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/server b/script/server index 716823c4..c4578701 100755 --- a/script/server +++ b/script/server @@ -8,7 +8,7 @@ cd "$(dirname "$0")/.." [ $WEB_CONCURRENCY ] || export WEB_CONCURRENCY=$(script/web_concurrency) [ $NGINX_WORKERS ] || export NGINX_WORKERS=$(script/web_concurrency --nginx) -echo "port=$PORT rack_env=$RACK_ENV web_concurrency=$WEB_CONCURRENCY nginx_workers=$NGINX_WORKERS" +echo "port=$PORT rack_env=$RACK_ENV web_concurrency=$WEB_CONCURRENCY nginx_workers=$NGINX_WORKERS" >&2 ruby="ruby -I lib -S" bexc="$ruby bundle exec" From 3e112294cb9e7e777c7452193aed17cb10953ab8 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 09:02:10 +0200 Subject: [PATCH 4/8] no double je --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index cfffe7c2..a9a67e8c 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ -web: bundle exec je ./script/server +web: ./script/server console: bundle exec je ./script/console sidekiq: bundle exec je sidekiq -c 4 -r ./lib/travis/sidekiq.rb -q build_cancellations, -q build_restarts, -q job_cancellations, -q job_restarts From 0890d0b76ab7219c2c147ca979246f16f0781a5b Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 09:06:42 +0200 Subject: [PATCH 5/8] fix heroku dyno detection --- script/web_concurrency | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/web_concurrency b/script/web_concurrency index 430b12dc..871e8076 100755 --- a/script/web_concurrency +++ b/script/web_concurrency @@ -15,7 +15,7 @@ end if ENV['DYNO'] # we're on Heroku - case `uname -u`.to_i + case `ulimit -u`.to_i when 256 then concurrency = 2 # 1x dyno when 512 then concurrency = 4 # 2x dyno when 32768 then concurrency = 16 # px dyno From 815d9e6e9bf56931a1c211663c571f2e6996ee7a Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 09:13:46 +0200 Subject: [PATCH 6/8] turn script/web_concurrency into a bash script, heroku did not like the ruby script --- script/web_concurrency | 47 +++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/script/web_concurrency b/script/web_concurrency index 871e8076..67556acd 100755 --- a/script/web_concurrency +++ b/script/web_concurrency @@ -1,33 +1,20 @@ -#!/usr/bin/env ruby --disable=gems --disable=rubyopt -GC.disable +#!/bin/bash -module System - extend self - def cpu_count - return Java::Java.lang.Runtime.getRuntime.availableProcessors if defined? Java::Java - return File.read('/proc/cpuinfo').scan(/^processor\s*:/).size if File.exist? '/proc/cpuinfo' - require 'win32ole' - WIN32OLE.connect("winmgmts://").ExecQuery("select * from Win32_ComputerSystem").NumberOfProcessors - rescue LoadError - Integer `sysctl -n hw.ncpu 2>/dev/null` rescue 1 - end -end +if [[ "$RACK_ENV" == "development" ]]; then + echo -n 2 + exit 0 +fi -if ENV['DYNO'] - # we're on Heroku - case `ulimit -u`.to_i - when 256 then concurrency = 2 # 1x dyno - when 512 then concurrency = 4 # 2x dyno - when 32768 then concurrency = 16 # px dyno - else $stderr.puts "Unkown dyno type, selecting concurrency of 4, because" - end -end +case $(ulimit -u) in + 256) echo -n 2; exit 0;; + 512) echo -n 4; exit 0;; + 32768) + if [[ "$1" == "--nginx" ]]; then + echo -n 4 + else + echo -n 16 + fi + exit 0;; +esac -case ENV['RACK_ENV'] || 'development' -when 'production' then concurrency ||= System.cpu_count -when 'development' then concurrency ||= 2 # use at least two so we can be sure things work concurrencly -else concurrency ||= 1 -end - -concurrency = System.cpu_count if ARGV[0] == '--nginx' and System.cpu_count < concurrency -print concurrency +echo -n 4 From 16c871d740a70a452ef0349c27db347c55a30318 Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 09:17:21 +0200 Subject: [PATCH 7/8] $tmp_dir is not set on heroku --- config/puma-config.rb | 3 ++- config/unicorn.rb | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/puma-config.rb b/config/puma-config.rb index 45527340..1c15535d 100644 --- a/config/puma-config.rb +++ b/config/puma-config.rb @@ -2,7 +2,8 @@ root = File.expand_path('../..', __FILE__) rackup "#{root}/config.ru" -bind "unix://#{ENV["tmp_dir"]}/nginx.socket" +tmp_dir = ENV.fetch("tmp_dir", "/tmp") +bind "unix://#{tmp_dir}/nginx.socket" environment ENV['RACK_ENV'] || 'development' threads 0, 16 diff --git a/config/unicorn.rb b/config/unicorn.rb index f71a3546..463a017c 100644 --- a/config/unicorn.rb +++ b/config/unicorn.rb @@ -3,7 +3,8 @@ worker_processes Integer(ENV.fetch('WEB_CONCURRENCY')) # amount of unicorn workers to spin up timeout 30 # restarts workers that hang for 30 seconds -listen File.expand_path("nginx.socket", ENV["tmp_dir"]), backlog: 1024 +tmp_dir = ENV.fetch("tmp_dir", "/tmp") +listen File.expand_path("nginx.socket", tmp_dir), backlog: 1024 require 'fileutils' before_fork do |server, worker| @@ -11,5 +12,5 @@ before_fork do |server, worker| require 'travis/api/app' # signal to nginx we're ready - FileUtils.touch("#{ENV["tmp_dir"]}/app-initialized") + FileUtils.touch("#{tmp_dir}/app-initialized") end From 6f0a42abee96ad12e37b9f363d58987c23b5811c Mon Sep 17 00:00:00 2001 From: Konstantin Haase Date: Fri, 9 Oct 2015 09:19:07 +0200 Subject: [PATCH 8/8] set tmp_dir in script/server --- bin/start-nginx | 7 ------- script/server | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/start-nginx b/bin/start-nginx index 1d0e3db4..b0d1f589 100755 --- a/bin/start-nginx +++ b/bin/start-nginx @@ -3,13 +3,6 @@ # make sure we kill all child processes once done trap '{ pkill -P $$; rm -f config/nginx.conf; exit 255; }' EXIT -if [[ $RACK_ENV == "production" ]]; then - export tmp_dir=/tmp -else - mkdir -p tmp - export tmp_dir=./tmp -fi - if [ -f bin/nginx ]; then nginx=bin/nginx else diff --git a/script/server b/script/server index c4578701..3afd1c27 100755 --- a/script/server +++ b/script/server @@ -13,6 +13,13 @@ echo "port=$PORT rack_env=$RACK_ENV web_concurrency=$WEB_CONCURRENCY nginx_worke ruby="ruby -I lib -S" bexc="$ruby bundle exec" +if [[ $RACK_ENV == "production" ]]; then + export tmp_dir=/tmp +else + mkdir -p tmp + export tmp_dir=./tmp +fi + cmd="unicorn config.ru -E $RACK_ENV -c config/unicorn.rb" [[ $RACK_ENV == "development" ]] && cmd="rerun -b -- $cmd" cmd="bin/start-nginx $bexec je $cmd"