track and enforce user-agent

Gemfile

@@ -25,6 +25,7 @@ gem 'pry'
 gem 'metriks',         '0.9.9.6'
 gem 'metriks-librato_metrics', github: 'eric/metriks-librato_metrics'
 gem 'micro_migrations'
+gem 'useragent'
 
 group :test do
   gem 'rspec',         '~> 2.13'

Gemfile.lock

@@ -36,7 +36,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-core.git
-  revision: 0f487381ecf7f06f759116b60a1e9e8544784286
+  revision: 0f7a43a373cd3a7236ed4b5d3ca4a312a4e4c656
   specs:
     travis-core (0.0.1)
       actionmailer (~> 3.2.19)
@@ -74,7 +74,7 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-yaml.git
-  revision: 121678eba7280b8269a73c56953cafd20e884bdc
+  revision: e899680992e31b25ddc0aad33d2189a7e588cc1f
   specs:
     travis-yaml (0.1.0)
 
@@ -321,6 +321,7 @@ GEM
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
+    useragent (0.10.0)
     uuidtools (2.1.5)
     virtus (1.0.3)
       axiom-types (~> 0.1)
@@ -362,4 +363,5 @@ DEPENDENCIES
   travis-support!
   travis-yaml!
   unicorn
+  useragent
   yard-sinatra!

lib/travis/api/app.rb

@@ -114,6 +114,7 @@ module Travis::Api
         use Travis::Api::App::Middleware::Logging
         use Travis::Api::App::Middleware::Metriks
         use Travis::Api::App::Middleware::Rewrite
+        use Travis::Api::App::Middleware::UserAgentTracker
 
         SettingsEndpoint.subclass :env_vars
         if Travis.config.endpoints.ssh_key

lib/travis/api/app/cors.rb

@@ -14,7 +14,7 @@ class Travis::Api::App
 
     options // do
       headers['Access-Control-Allow-Methods'] = "HEAD, GET, POST, PATCH, PUT, DELETE"
-      headers['Access-Control-Allow-Headers'] = "Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since"
+      headers['Access-Control-Allow-Headers'] = "Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since, X-User-Agent"
     end
   end
 end

lib/travis/api/app/middleware/user_agent_tracker.rb

@@ -0,0 +1,85 @@
+require 'travis/api/app'
+require 'useragent'
+
+class Travis::Api::App
+  class Middleware
+    class UserAgentTracker < Middleware
+      WEB_BROWSERS = [
+        "Internet Explorer",
+        "Webkit", "Chrome", "Safari", "Android",
+        "Firefox", "Camino", "Iceweasel", "Seamonkey", "Android",
+        "Opera", "Mozilla"
+      ]
+
+      before(agent: /^$/) do
+        ::Metriks.meter("api.user_agent.missing").mark
+        halt(400, "error" => "missing User-Agent header") if Travis::Features.feature_active?(:require_user_agent)
+      end
+
+      before(agent: /^.+$/) do
+        agent = UserAgent.parse(request.user_agent)
+        case agent.browser
+        when *WEB_BROWSERS                   then mark_browser
+        when "curl", "Wget"                  then mark(:console, agent.browser)
+        when "travis-api-wrapper"            then mark(:script, :node_js, agent.browser)
+        when "TravisPy"                      then mark(:script, :python,  agent.browser)
+        when "Ruby", "PHP", "Perl", "Python" then mark(:script, agent.browser, :vanilla)
+        when "Faraday"                       then mark(:script, :ruby, :vanilla)
+        when "Travis"                        then mark_travis(agent)
+        else mark_unknown
+        end
+      end
+
+      def mark_browser
+        # allows a JavaScript Client to set X-User-Agent, for instance to "travis-web" in travis-web
+        x_agent = UserAgent.parse(env['HTTP_X_USER_AGENT'] || 'unknown').browser
+        mark(:browser, x_agent)
+      end
+
+      def mark_travis(agent)
+        os, *rest               = agent.application.comment
+        ruby, rubygems, command = "unknown", "unknown", nil
+
+        rest.each do |comment|
+          case comment
+          when /^Ruby (\d\.\d.\d)/ then ruby     = $1
+          when /^RubyGems (.+)$/   then rubygems = $1
+          when /^command (.+)$/    then command  = $1
+          end
+        end
+
+        # "Ubuntu 12.04 like Linux" => "linux.ubuntu.12.04"
+        if os =~ /^(.+) (\S+) like (\S+)$/
+          os = "#{$3}.#{$1}.#{$2[/\d+\.\d+/]}"
+        end
+
+        if command
+          mark(:cli, version: agent.version, ruby: ruby, rubygems: rubygems, command: command, os: os)
+        else
+          # only track ruby version and library version for non-cli usage
+          mark(:script, :ruby, :travis, version: agent.version, ruby: ruby)
+        end
+      end
+
+      def mark_unknown
+        logger.warn "[user-agent-tracker] Unknown User-Agent: %p" % request.user_agent
+        mark(:unknown)
+      end
+
+      def track_key(string)
+        string.to_s.downcase.gsub(/[^a-z0-9\-\.]+/, '_')
+      end
+
+      def mark(*keys)
+        key = "api.user_agent"
+        keys.each do |subkey|
+          if subkey.is_a? Hash
+            subkey.each_pair { |k, v| ::Metriks.meter("#{key}.#{track_key(k)}.#{track_key(v)}").mark }
+          else
+            ::Metriks.meter(key << "." << track_key(subkey)).mark
+          end
+        end
+      end
+    end
+  end
+end

spec/unit/cors_spec.rb

@@ -44,7 +44,7 @@ describe Travis::Api::App::Cors do
     end
 
     it 'sets Access-Control-Allow-Headers' do
-      headers['Access-Control-Allow-Headers'].should == "Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since"
+      headers['Access-Control-Allow-Headers'].should == "Content-Type, Authorization, Accept, If-None-Match, If-Modified-Since, X-User-Agent"
     end
   end
 end

spec/unit/middleware/user_agent_tracker_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+
+describe Travis::Api::App::Middleware::UserAgentTracker do
+  before do
+    mock_app do
+      use Travis::Api::App::Middleware::UserAgentTracker
+      get('/') { 'ok' }
+    end
+  end
+
+  def expect_meter(name)
+    Metriks.expects(:meter).with(name).returns(stub("meter", mark: nil))
+  end
+
+  def get(env = {})
+    env['HTTP_USER_AGENT'] ||= agent if agent
+    super('/', {}, env)
+  end
+
+  context 'missing User-Agent' do
+    let(:agent) { }
+
+    it "tracks it" do
+      expect_meter("api.user_agent.missing")
+      get.should be_ok
+    end
+
+    it "denies request if require_user_agent feature is enabled" do
+      Travis::Features.expects(:feature_active?).with(:require_user_agent).returns(true)
+      get.status.should be == 400
+    end
+  end
+
+  context 'web browser' do
+    let(:agent) { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.36 Safari/537.36" }
+
+    specify 'without X-User-Agent' do
+      expect_meter("api.user_agent.browser")
+      expect_meter("api.user_agent.browser.unknown")
+      get
+    end
+
+    specify 'with X-User-Agent' do
+      expect_meter("api.user_agent.browser")
+      expect_meter("api.user_agent.browser.travis-web")
+      get('HTTP_X_USER_AGENT' => 'travis-web')
+    end
+  end
+
+  context 'console' do
+    let(:agent) { 'curl' }
+    specify do
+      expect_meter("api.user_agent.console")
+      expect_meter("api.user_agent.console.curl")
+      get
+    end
+  end
+
+  context 'travis-api-wrapper' do
+    let(:agent) { 'travis-api-wrapper - v0.01 - (cmaujean@gmail.com)' }
+    specify do
+      expect_meter("api.user_agent.script")
+      expect_meter("api.user_agent.script.node_js")
+      expect_meter("api.user_agent.script.node_js.travis-api-wrapper")
+      get
+    end
+  end
+
+  context 'TravisPy' do
+    let(:agent) { 'TravisPy' }
+    specify do
+      expect_meter("api.user_agent.script")
+      expect_meter("api.user_agent.script.python")
+      expect_meter("api.user_agent.script.python.travispy")
+      get
+    end
+  end
+
+  context 'Ruby' do
+    let(:agent) { 'Ruby' }
+    specify do
+      expect_meter("api.user_agent.script")
+      expect_meter("api.user_agent.script.ruby")
+      expect_meter("api.user_agent.script.ruby.vanilla")
+      get
+    end
+  end
+
+  context 'Faraday' do
+    let(:agent) { 'Faraday' }
+    specify do
+      expect_meter("api.user_agent.script")
+      expect_meter("api.user_agent.script.ruby")
+      expect_meter("api.user_agent.script.ruby.vanilla")
+      get
+    end
+  end
+
+  context 'travis.rb' do
+    let(:agent) { 'Travis/1.6.8 (Mac OS X 10.9.2 like Darwin; Ruby 2.1.1p42; RubyGems 2.0.14) Faraday/0.8.9 Typhoeus/0.6.7' }
+    specify do
+      expect_meter("api.user_agent.script")
+      expect_meter("api.user_agent.script.ruby")
+      expect_meter("api.user_agent.script.ruby.travis")
+      expect_meter("api.user_agent.script.ruby.travis.version.1.6.8")
+      expect_meter("api.user_agent.script.ruby.travis.ruby.2.1.1")
+      get
+    end
+  end
+
+  context 'Travis CLI' do
+    let(:agent) { 'Travis/1.6.8 (Mac OS X 10.10.2 like Darwin; Ruby 2.1.1; RubyGems 2.0.14; command whoami) Faraday/0.8.9 Typhoeus/0.6.7' }
+    specify do
+      expect_meter("api.user_agent.cli")
+      expect_meter("api.user_agent.cli.version.1.6.8")
+      expect_meter("api.user_agent.cli.ruby.2.1.1")
+      expect_meter("api.user_agent.cli.rubygems.2.0.14")
+      expect_meter("api.user_agent.cli.command.whoami")
+      expect_meter("api.user_agent.cli.os.darwin.mac_os_x.10.10")
+      get
+    end
+  end
+end

travis-api.gemspec

@@ -18,17 +18,18 @@ Gem::Specification.new do |s|
     "Henrik Hodne",
     "Andre Arko",
     "Erik Michaels-Ober",
-    "Brian Ford",
     "Steve Richert",
-    "rainsun",
+    "Brian Ford",
-    "Bryan Goldstein",
-    "James Dennes",
-    "Nick Schonning",
     "Patrick Williams",
+    "Bryan Goldstein",
     "Puneeth Chaganti",
     "Thais Camilo and Konstantin Haase",
     "Tim Carey-Smith",
-    "Zachary Scott"
+    "Zachary Scott",
+    "James Dennes",
+    "rainsun",
+    "Dan Rice",
+    "Nick Schonning"
   ]
 
   s.email = [
@@ -44,17 +45,19 @@ Gem::Specification.new do |s|
     "andre@arko.net",
     "svenfuchs@artweb-design.de",
     "sferik@gmail.com",
-    "bford@engineyard.com",
+    "henrik@travis-ci.com",
     "steve.richert@gmail.com",
-    "patrick@bittorrent.com",
+    "bford@engineyard.com",
-    "punchagan@muse-amuse.in",
-    "jdennes@gmail.com",
     "nschonni@gmail.com",
+    "brysgo@gmail.com",
+    "punchagan@muse-amuse.in",
+    "e@zzak.io",
+    "jdennes@gmail.com",
     "rainsuner@gmail.com",
     "dev+narwen+rkh@rkh.im",
     "tim@spork.in",
-    "e@zzak.io",
+    "dan@zoombody.com",
-    "brysgo@gmail.com"
+    "patrick@bittorrent.com"
   ]
 
   s.files = [
@@ -65,7 +68,6 @@ Gem::Specification.new do |s|
     "bin/start-nginx",
     "config.ru",
     "config/database.yml",
-    "config/nginx.conf.erb",
     "config/puma-config.rb",
     "config/unicorn.rb",
     "lib/tasks/build_update_branch.rake",
@@ -83,6 +85,7 @@ Gem::Specification.new do |s|
     "lib/travis/api/app/endpoint/builds.rb",
     "lib/travis/api/app/endpoint/documentation.rb",
     "lib/travis/api/app/endpoint/endpoints.rb",
+    "lib/travis/api/app/endpoint/env_vars.rb",
     "lib/travis/api/app/endpoint/home.rb",
     "lib/travis/api/app/endpoint/hooks.rb",
     "lib/travis/api/app/endpoint/jobs.rb",
@@ -91,6 +94,7 @@ Gem::Specification.new do |s|
     "lib/travis/api/app/endpoint/repos.rb",
     "lib/travis/api/app/endpoint/requests.rb",
     "lib/travis/api/app/endpoint/setting_endpoint.rb",
+    "lib/travis/api/app/endpoint/singleton_settings_endpoint.rb",
     "lib/travis/api/app/endpoint/uptime.rb",
     "lib/travis/api/app/endpoint/users.rb",
     "lib/travis/api/app/extensions.rb",
@@ -119,6 +123,7 @@ Gem::Specification.new do |s|
     "lib/travis/api/app/responders/plain.rb",
     "lib/travis/api/app/responders/service.rb",
     "lib/travis/api/app/responders/xml.rb",
+    "lib/travis/api/app/services/schedule_request.rb",
     "lib/travis/api/serializer.rb",
     "lib/travis/api/v2.rb",
     "lib/travis/api/v2/http.rb",
@@ -143,11 +148,13 @@ Gem::Specification.new do |s|
     "lib/travis/api/v2/http/request.rb",
     "lib/travis/api/v2/http/requests.rb",
     "lib/travis/api/v2/http/ssh_key.rb",
-    "lib/travis/api/v2/http/ssh_keys.rb",
     "lib/travis/api/v2/http/ssl_key.rb",
     "lib/travis/api/v2/http/user.rb",
     "lib/travis/api/v2/http/validation_error.rb",
+    "lib/travis/private_key.rb",
     "public/favicon.ico",
+    "public/images/result/canceled.png",
+    "public/images/result/canceled.svg",
     "public/images/result/error.png",
     "public/images/result/error.svg",
     "public/images/result/failing.png",
@@ -159,12 +166,14 @@ Gem::Specification.new do |s|
     "public/images/result/unknown.png",
     "public/images/result/unknown.svg",
     "script/console",
+    "script/repos_stats.rb",
     "script/server",
     "spec/integration/formats_handling_spec.rb",
     "spec/integration/responders_spec.rb",
     "spec/integration/routes.backup.rb",
     "spec/integration/scopes_spec.rb",
     "spec/integration/settings_endpoint_spec.rb",
+    "spec/integration/singleton_settings_endpoint_spec.rb",
     "spec/integration/uptime_spec.rb",
     "spec/integration/v1/branches_spec.rb",
     "spec/integration/v1/builds_spec.rb",
@@ -179,6 +188,7 @@ Gem::Specification.new do |s|
     "spec/integration/v2/repositories_spec.rb",
     "spec/integration/v2/requests_spec.rb",
     "spec/integration/v2/settings/env_vars_spec.rb",
+    "spec/integration/v2/settings/ssh_key_spec.rb",
     "spec/integration/v2/users_spec.rb",
     "spec/integration/v2_spec.backup.rb",
     "spec/integration/version_spec.rb",
@@ -194,6 +204,7 @@ Gem::Specification.new do |s|
     "spec/unit/api/v2/http/build_spec.rb",
     "spec/unit/api/v2/http/builds_spec.rb",
     "spec/unit/api/v2/http/caches_spec.rb",
+    "spec/unit/api/v2/http/env_var_spec.rb",
     "spec/unit/api/v2/http/hooks_spec.rb",
     "spec/unit/api/v2/http/job_spec.rb",
     "spec/unit/api/v2/http/jobs_spec.rb",
@@ -219,6 +230,7 @@ Gem::Specification.new do |s|
     "spec/unit/endpoint/lint_spec.rb",
     "spec/unit/endpoint/logs_spec.rb",
     "spec/unit/endpoint/repos_spec.rb",
+    "spec/unit/endpoint/requests_spec.rb",
     "spec/unit/endpoint/users_spec.rb",
     "spec/unit/endpoint_spec.rb",
     "spec/unit/extensions/expose_pattern_spec.rb",