Merge branch 'master' into caches_endpoint

'

Gemfile

@@ -37,7 +37,7 @@ gem 'jemalloc'
 gem 'customerio'
 
 group :development, :test do
-  gem 'travis-migrations', github: 'travis-ci/travis-migrations'
+  gem 'travis-migrations', github: 'travis-ci/travis-migrations', branch: 'add_travis_logs_structure'
 end
 
 group :test do
@@ -47,6 +47,7 @@ group :test do
   gem 'mocha',         '~> 0.12'
   gem 'database_cleaner', '~> 0.8.0'
   gem 'timecop',       '~> 0.8.0'
+  gem 'webmock'
 end
 
 group :development do

Gemfile.lock

@@ -41,7 +41,8 @@ GIT
 
 GIT
   remote: git://github.com/travis-ci/travis-migrations.git
-  revision: dc432e45354287c617c3ae07a72e9e3c4be012cd
+  revision: cfc5b94d9380d945d42e743beb0bc2433d846c93
+  branch: add_travis_logs_structure
   specs:
     travis-migrations (0.0.2)
 
@@ -146,6 +147,8 @@ GEM
       activerecord (~> 3.2.0, >= 3.2.9)
     concurrent-ruby (1.0.2)
     connection_pool (2.2.0)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     customerio (1.0.0)
       multi_json (~> 1.0)
     dalli (2.7.6)
@@ -189,6 +192,7 @@ GEM
       multi_json (~> 1.11)
       os (~> 0.9)
       signet (~> 0.7)
+    hashdiff (0.3.0)
     hashr (0.0.22)
     hike (1.2.3)
     hitimes (1.2.4)
@@ -288,6 +292,7 @@ GEM
       rspec-expectations (>= 2.99.0.beta1)
     rspec-mocks (2.99.4)
     ruby_dep (1.3.1)
+    safe_yaml (1.0.4)
     sentry-raven (1.0.0)
       faraday (>= 0.7.6)
     sidekiq (4.1.2)
@@ -345,6 +350,10 @@ GEM
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
       equalizer (~> 0.0, >= 0.0.9)
+    webmock (2.1.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
     yard (0.8.7.6)
 
 PLATFORMS
@@ -392,6 +401,7 @@ DEPENDENCIES
   travis-support!
   travis-yaml!
   unicorn
+  webmock
   yard-sinatra!
 
 BUNDLED WITH

README.md

@@ -38,7 +38,7 @@ NB detail for how `rake` sets up the database can be found in the `Rakefile`. In
 $ RAILS_ENV=development bundle exec rake db:create
 $ RAILS_ENV=test bundle exec rake db:create
 ```
-#### Optional
+#### Travis Logs DB setup
 Clone `travis-logs` and copy the `logs` database (assume the PostgreSQL user is `postgres`):
 ```sh-session
 $ cd ..

Rakefile

@@ -4,7 +4,9 @@ namespace :db do
     desc "Create and migrate the #{env} database"
     task :create do
       sh "createdb travis_#{env}" rescue nil
+      sh "createdb travis_logs_#{env}" rescue nil
       sh "psql -q travis_#{env} < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/structure.sql"
+      sh "psql -q travis_logs_#{env} < #{Gem.loaded_specs['travis-migrations'].full_gem_path}/db/travis_logs_structure.sql"
     end
   end
 end

lib/travis/api/v3/models/log.rb

@@ -1,5 +1,6 @@
 module Travis::API::V3
   class Models::Log < Model
+    establish_connection(Travis.config.logs_database)
     belongs_to :job
     belongs_to :removed_by, class_name: 'User', foreign_key: :removed_by
     has_many  :log_parts, dependent: :destroy

lib/travis/api/v3/models/log_part.rb

@@ -1,5 +1,6 @@
 module Travis::API::V3
   class Models::LogPart < Model
+    establish_connection(Travis.config.logs_database)
     belongs_to :log
   end
 end

lib/travis/api/v3/queries/log.rb

@@ -0,0 +1,35 @@
+module Travis::API::V3
+  class Queries::Log < Query
+    require 'net/http'
+    require 'uri'
+
+    def find(job)
+      #check for the log in the Logs DB
+      log = Models::Log.find_by_job_id(job.id)
+
+      raise EntityMissing, 'log not found'.freeze if log.nil?
+      #if the log has been archived, go to s3
+      if log.archived_at
+        ## if it's not there then fetch it from S3, and return it wrapped as a compatible log_parts object with a hard coded #number (log_parts have a number) and the parts chunked (not sure how to do this)
+        archived_log_path = archive_url("/jobs/#{job.id}/log.txt")
+        content = Net::HTTP.get(URI.parse(archived_log_path))
+        log_parts = []
+
+        content.each_line.with_index do |line, number|
+          log_part = Models::LogPart.new(log_id: log.id, content: line.chomp, number: number, created_at: log.created_at)
+          log_parts << log_part
+        end
+        log.log_parts = log_parts
+      end
+      log
+    end
+
+    def archive_url(path)
+      "https://s3.amazonaws.com/#{hostname('archive')}#{path}"
+    end
+
+    def hostname(name)
+      "#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}"
+    end
+  end
+end

lib/travis/api/v3/renderer/log.rb

@@ -0,0 +1,6 @@
+module Travis::API::V3
+  class Renderer::Log < Renderer::ModelRenderer
+    representation(:minimal, :id, :content)
+    representation(:standard, *representations[:minimal], :log_parts)
+  end
+end

lib/travis/api/v3/renderer/log_part.rb

@@ -0,0 +1,6 @@
+module Travis::API::V3
+  class Renderer::LogPart < Renderer::ModelRenderer
+    representation(:minimal, :content, :number)
+    representation(:standard, *representations[:minimal], :log)
+  end
+end

lib/travis/api/v3/renderer/log_parts.rb

@@ -0,0 +1,6 @@
+module Travis::API::V3
+  class Renderer::LogParts < Renderer::CollectionRenderer
+    type           :log_parts
+    collection_key :log_parts
+  end
+end

lib/travis/api/v3/routes.rb

@@ -43,6 +43,11 @@ module Travis::API::V3
       post :cancel, '/cancel'
       post :restart, '/restart'
       post :debug, '/debug'
+
+      resource :log do
+        route '/log'
+        get   :find
+      end
     end
 
     resource :lint do

lib/travis/api/v3/services.rb

@@ -16,6 +16,7 @@ module Travis::API::V3
     Job           = Module.new { extend Services }
     Jobs          = Module.new { extend Services }
     Lint          = Module.new { extend Services }
+    Log           = Module.new { extend Services }
     Organization  = Module.new { extend Services }
     Organizations = Module.new { extend Services }
     Owner         = Module.new { extend Services }

lib/travis/api/v3/services/log/find.rb

@@ -0,0 +1,8 @@
+module Travis::API::V3
+  class Services::Log::Find < Service
+    def run!
+      job = check_login_and_find(:job)
+      query.find(job)
+    end
+  end
+end

lib/travis/config/defaults.rb

@@ -18,6 +18,7 @@ module Travis
             assets:        { host: HOSTS[Travis.env.to_sym] },
             amqp:          { username: 'guest', password: 'guest', host: 'localhost', prefetch: 1 },
             database:      { adapter: 'postgresql', database: "travis_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } },
+            logs_database: { adapter: 'postgresql', database: "travis_logs_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } },
             s3:            { access_key_id: '', secret_access_key: '' },
             pusher:        { app_id: 'app-id', key: 'key', secret: 'secret' },
             sidekiq:       { namespace: 'sidekiq', pool_size: 1 },

spec/lib/travis/config_spec.rb

@@ -73,6 +73,16 @@ describe Travis::Config do
         :variables => { :statement_timeout => 10000 }
       }
     end
+
+    it 'logs database' do
+      config.logs_database.should == {
+        :adapter => 'postgresql',
+        :database => 'travis_logs_test',
+        :encoding => 'unicode',
+        :min_messages => 'warning',
+        :variables => { :statement_timeout => 10000 }
+      }
+    end
   end
 
   describe 'resource urls' do

spec/spec_helper.rb

@@ -10,6 +10,7 @@ require 'gh'
 require 'multi_json'
 require 'pry'
 require 'stackprof'
+require 'webmock/rspec'
 
 require 'travis/api/app'
 require 'travis/testing'

spec/v3/services/log/find_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+
+describe Travis::API::V3::Services::Log::Find, set_app: true do
+  let(:user)        { Factory.create(:user) }
+  let(:repo)        { Factory.create(:repository, owner_name: user.login, name: 'minimal', owner: user)}
+  let(:build)       { Factory.create(:build, repository: repo) }
+  let(:job)         { Travis::API::V3::Models::Job.create(build: build) }
+  let(:job2)        { Travis::API::V3::Models::Job.create(build: build)}
+  let(:s3job)       { Travis::API::V3::Models::Job.create(build: build) }
+  let(:token)       { Travis::Api::App::AccessToken.create(user: user, app_id: 1) }
+  let(:headers)     { { 'HTTP_AUTHORIZATION' => "token #{token}" } }
+  let(:parsed_body) { JSON.load(body) }
+  let(:log)         { Travis::API::V3::Models::Log.create(job: job) }
+  let(:log2)        { Travis::API::V3::Models::Log.create(job: job2) }
+  let(:s3log)       { Travis::API::V3::Models::Log.create(job: s3job, content: 'minimal log 1') }
+
+  before { Travis::API::V3::AccessControl::LegacyToken.any_instance.stubs(:visible?).returns(true) }
+
+  context 'when log stored in db' do
+    describe 'returns log with an array of Log Parts' do
+      example do
+        log_part = log.log_parts.create(content: "logging it", number: 0)
+        get("/v3/job/#{log.job.id}/log", {}, headers)
+        expect(parsed_body).to eq(
+          '@href' => "/v3/job/#{log.job.id}/log",
+          '@representation' => 'standard',
+          '@type' => 'log',
+          'content' => nil,
+          'id' => log.id,
+          'log_parts'       => [{
+          "@type"           => "log_part",
+          "@representation" => "minimal",
+          "content"         => log_part.content,
+          "number"          => log_part.number }])
+      end
+    end
+    describe 'returns log as plain text'
+  end
+
+  context 'when log not found in db but stored on S3' do
+    describe 'returns log with an array of Log Parts' do
+      before do
+        stub_request(:get, "https://s3.amazonaws.com/archive.travis-ci.org/jobs/#{s3job.id}/log.txt").
+         with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'Host'=>'s3.amazonaws.com', 'User-Agent'=>'Ruby'}).
+         to_return(:status => 200, :body => "$ git clean -fdx\nRemoving Gemfile.lock\n$ git fetch", :headers => {})
+      end
+      example do
+        s3log.update_attributes(archived_at: Time.now)
+        get("/v3/job/#{s3job.id}/log", {}, headers)
+
+        expect(parsed_body).to eq(
+          '@href' => "/v3/job/#{s3job.id}/log",
+          '@representation' => 'standard',
+          '@type' => 'log',
+          'content' => 'minimal log 1',
+          'id' => s3log.id,
+          'log_parts'       => [{
+            "@type"=>"log_part",
+            "@representation"=>"minimal",
+            "content"=>"$ git clean -fdx",
+            "number"=>0}, {
+            "@type"=>"log_part",
+            "@representation"=>"minimal",
+            "content"=>"Removing Gemfile.lock",
+            "number"=>1}, {
+            "@type"=>"log_part",
+            "@representation"=>"minimal",
+            "content"=>"$ git fetch",
+            "number"=>2}])
+      end
+    end
+    describe 'returns log as plain text'
+  end
+
+  context 'when log not found anywhere' do
+    describe 'does not return log - returns error' do
+      before { log.delete }
+      example do
+        get("/v3/job/#{job.id}/log", {}, headers)
+        expect(parsed_body).to eq({
+          "@type"=>"error",
+          "error_type"=>"not_found",
+          "error_message"=>"log not found"})
+        end
+    end
+  end
+
+  context 'when log removed by user' do
+    describe 'does not return log'
+  end
+end