From d0a143d76a366b5b0a21b197d6da9d1a300eeea5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 19:44:33 -0400
Subject: [PATCH 01/11] ensure that the Time.now builds get started and tag the
 Sentry errors from cron

---
 lib/travis/api/v3/models/cron.rb   |  2 +-
 lib/travis/api/v3/queries/crons.rb |  3 ++-
 spec/v3/models/cron_spec.rb        | 22 ++++++++++++++--------
 spec/v3/queries/cron_spec.rb       |  2 +-
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/lib/travis/api/v3/models/cron.rb b/lib/travis/api/v3/models/cron.rb
index 68f664f2..e86527a9 100644
--- a/lib/travis/api/v3/models/cron.rb
+++ b/lib/travis/api/v3/models/cron.rb
@@ -13,7 +13,7 @@ module Travis::API::V3
       elsif last_cron_build_date >= planned_time(LastBuild)
         planned_time(ThisBuild)
       else
-        Time.now
+        Time.now - 5.minutes
       end
     end
 
diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index d5799d58..b5daefdd 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -7,10 +7,11 @@ module Travis::API::V3
 
     def start_all()
       Models::Cron.all.select do |cron|
+        @cron = cron
         start(cron) if cron.next_enqueuing <= Time.now
       end
       rescue => e
-        Raven.capture_exception(e)
+        Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
     end
 
     def start(cron)
diff --git a/spec/v3/models/cron_spec.rb b/spec/v3/models/cron_spec.rb
index 9e323e66..7b9dd93a 100644
--- a/spec/v3/models/cron_spec.rb
+++ b/spec/v3/models/cron_spec.rb
@@ -43,11 +43,13 @@ describe Travis::API::V3::Models::Cron do
   describe "push build is ignored if disable by build is false" do
 
     before do
+      Timecop.return
       Timecop.travel(DateTime.new(2015, 12, 31, 16))
     end
 
     after do
       Timecop.return
+      Timecop.freeze(Time.now.utc)
     end
 
     it "for daily builds" do
@@ -85,11 +87,13 @@ describe Travis::API::V3::Models::Cron do
   describe "disable by build works with build" do
 
     before do
+      Timecop.return
       Timecop.travel(DateTime.new(2015, 12, 31, 16))
     end
 
     after do
       Timecop.return
+      Timecop.freeze(Time.now.utc)
     end
 
     it "for daily builds" do
@@ -121,11 +125,13 @@ describe Travis::API::V3::Models::Cron do
   describe "disable by build works without build" do
 
     before do
+      Timecop.return
       Timecop.travel(DateTime.new(2015, 12, 31, 16))
     end
 
     after do
       Timecop.return
+      Timecop.freeze(Time.now.utc)
     end
 
     it "for daily builds" do
@@ -157,12 +163,12 @@ describe Travis::API::V3::Models::Cron do
   describe "build starts now if next build time is in the past" do
 
     before do
-      # nothing, this time
-      # time freeze is performed in examples
+      Timecop.return
     end
 
     after do
       Timecop.return
+      Timecop.freeze(Time.now.utc)
     end
 
     it "for daily builds with disable_by_build true" do
@@ -170,7 +176,7 @@ describe Travis::API::V3::Models::Cron do
       cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: true)
       build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron')
       Timecop.freeze(DateTime.new(2016, 1, 1, 19))
-      expect(cron.next_enqueuing).to be == DateTime.now
+      expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes
       build.destroy
       cron.destroy
     end
@@ -180,7 +186,7 @@ describe Travis::API::V3::Models::Cron do
       cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'daily', disable_by_build: false)
       build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron')
       Timecop.freeze(DateTime.new(2016, 1, 1, 19))
-      expect(cron.next_enqueuing).to be == DateTime.now
+      expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes
       build.destroy
       cron.destroy
     end
@@ -190,7 +196,7 @@ describe Travis::API::V3::Models::Cron do
       cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: true)
       build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron')
       Timecop.freeze(DateTime.new(2016, 1, 7, 19))
-      expect(cron.next_enqueuing).to be == DateTime.now
+      expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes
       build.destroy
       cron.destroy
     end
@@ -200,7 +206,7 @@ describe Travis::API::V3::Models::Cron do
       cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'weekly', disable_by_build: false)
       build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron')
       Timecop.freeze(DateTime.new(2016, 1, 7, 19))
-      expect(cron.next_enqueuing).to be == DateTime.now
+      expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes
       build.destroy
       cron.destroy
     end
@@ -210,7 +216,7 @@ describe Travis::API::V3::Models::Cron do
       cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: true)
       build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron')
       Timecop.freeze(DateTime.new(2016, 1, 31, 19))
-      expect(cron.next_enqueuing).to be == DateTime.now
+      expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes
       build.destroy
       cron.destroy
     end
@@ -220,7 +226,7 @@ describe Travis::API::V3::Models::Cron do
       cron = Travis::API::V3::Models::Cron.create(branch_id: branch.id, interval: 'monthly', disable_by_build: false)
       build = Travis::API::V3::Models::Build.create(:repository_id => repo.id, :branch_name => branch.name, :event_type => 'cron')
       Timecop.freeze(DateTime.new(2016, 1, 31, 19))
-      expect(cron.next_enqueuing).to be == DateTime.now
+      expect(cron.next_enqueuing).to be == DateTime.now - 5.minutes
       build.destroy
       cron.destroy
     end
diff --git a/spec/v3/queries/cron_spec.rb b/spec/v3/queries/cron_spec.rb
index d6d16889..7013495b 100644
--- a/spec/v3/queries/cron_spec.rb
+++ b/spec/v3/queries/cron_spec.rb
@@ -26,7 +26,7 @@ describe Travis::API::V3::Queries::Crons do
       cron = Travis::API::V3::Models::Cron.create(branch_id: existing_branch.id, interval: 'daily', disable_by_build: false)
       error = StandardError.new('Konstantin broke all the thingz!')
       Travis::API::V3::Models::Cron.any_instance.stubs(:branch).raises(error)
-      Raven.expects(:capture_exception).with(error)
+      Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron.id })
       query.start_all
     end
   end

From 6ca063cf3dbf6fe611ae4dd90952574c82c8fcfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 21:26:04 -0400
Subject: [PATCH 02/11] adding an error to see if I get it from Sentry

---
 lib/travis/api/v3/queries/crons.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index b5daefdd..dfa730a5 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -8,6 +8,7 @@ module Travis::API::V3
     def start_all()
       Models::Cron.all.select do |cron|
         @cron = cron
+        bad_things
         start(cron) if cron.next_enqueuing <= Time.now
       end
       rescue => e

From d41ddd7171af89b71fbc81799a200c29264a04c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 22:03:31 -0400
Subject: [PATCH 03/11] testing console on staging

---
 lib/travis/api/app.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb
index d61a498b..fe1e8e7b 100644
--- a/lib/travis/api/app.rb
+++ b/lib/travis/api/app.rb
@@ -195,7 +195,7 @@ module Travis::Api
           end
         end
 
-        if use_monitoring? && !console?
+        if use_monitoring? #&& !console?
           setup_monitoring
         end
       end

From 20d4df8af34e066b0d49389b92ea8de7ad95af1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 22:34:49 -0400
Subject: [PATCH 04/11] test output for staging

---
 lib/travis/api/v3/queries/crons.rb | 3 +++
 lib/travis/api/v3/service.rb       | 2 +-
 script/start_crons                 | 4 ++++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index dfa730a5..e1b5af1b 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -6,8 +6,11 @@ module Travis::API::V3
     end
 
     def start_all()
+      puts "starting #{Models::Cron.count} crons."
       Models::Cron.all.select do |cron|
         @cron = cron
+        puts cron.next_enqueuing
+        puts Time.now
         bad_things
         start(cron) if cron.next_enqueuing <= Time.now
       end
diff --git a/lib/travis/api/v3/service.rb b/lib/travis/api/v3/service.rb
index f3243189..7a87f44c 100644
--- a/lib/travis/api/v3/service.rb
+++ b/lib/travis/api/v3/service.rb
@@ -68,7 +68,7 @@ module Travis::API::V3
 
     def check_login_and_find(*args)
       raise LoginRequired unless access_control.full_access_or_logged_in?
-      find(*args) or raise NotFound 
+      find(*args) or raise NotFound
     end
 
     def not_found(actually_not_found = false, type = nil)
diff --git a/script/start_crons b/script/start_crons
index 63a0273e..13e69b46 100755
--- a/script/start_crons
+++ b/script/start_crons
@@ -6,4 +6,8 @@ require 'travis/api/app'
 
 Travis::Api::App.setup
 
+puts "start"
+
 Travis::API::V3::Services::Crons::Start.new(nil,nil,nil).run!
+
+puts "done"

From 1788e0c0fb02419a9603ff4e55ab6286f3160739 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 22:42:32 -0400
Subject: [PATCH 05/11] more output for the strange behavior on staging

---
 lib/travis/api/v3/queries/crons.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index e1b5af1b..2273f4cb 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -15,7 +15,9 @@ module Travis::API::V3
         start(cron) if cron.next_enqueuing <= Time.now
       end
       rescue => e
-        Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
+        puts "bad things happened"
+        puts e.inspect
+        puts Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
     end
 
     def start(cron)

From d42362eacc761f8d3aa2391120428cc38cbf7c7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 22:49:02 -0400
Subject: [PATCH 06/11] looks like Raven isn't getting the reports because the
 dyno doesn't exist long enough for the http request to get out :(

---
 lib/travis/api/v3/queries/crons.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index 2273f4cb..561f269b 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -18,6 +18,7 @@ module Travis::API::V3
         puts "bad things happened"
         puts e.inspect
         puts Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
+        sleep(10)
     end
 
     def start(cron)

From 0dc7e2743c4e43984ea3b78d46e53730c38a3c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 22:56:13 -0400
Subject: [PATCH 07/11] take intentional error out to see real errors

---
 lib/travis/api/v3/queries/crons.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index 561f269b..a0420ad3 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -11,7 +11,6 @@ module Travis::API::V3
         @cron = cron
         puts cron.next_enqueuing
         puts Time.now
-        bad_things
         start(cron) if cron.next_enqueuing <= Time.now
       end
       rescue => e

From abe7b13605ef085300d9e25ff29fbe96ef75a138 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 23:16:00 -0400
Subject: [PATCH 08/11] better output

---
 lib/travis/api/v3/queries/crons.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index a0420ad3..c6ede5e8 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -6,12 +6,12 @@ module Travis::API::V3
     end
 
     def start_all()
-      puts "starting #{Models::Cron.count} crons."
+      puts "reviewing #{Models::Cron.count} crons."
       Models::Cron.all.select do |cron|
         @cron = cron
-        puts cron.next_enqueuing
-        puts Time.now
-        start(cron) if cron.next_enqueuing <= Time.now
+        ne = cron.next_enqueuing
+        puts "Next enqueuing: #{ne}, time now: #{Time.now}, will it run? #{ne <= Time.now}"
+        start(cron) if ne <= Time.now
       end
       rescue => e
         puts "bad things happened"

From 779b6a569ebaafc27b7cefafc3c00419205f7657 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Tue, 2 Aug 2016 23:49:47 -0400
Subject: [PATCH 09/11] don't allow one error to stop all crons from running

---
 lib/travis/api/v3/queries/crons.rb | 19 ++++++++++---------
 spec/v3/queries/cron_spec.rb       | 15 +++++++++++++++
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index c6ede5e8..f848b6e3 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -8,16 +8,17 @@ module Travis::API::V3
     def start_all()
       puts "reviewing #{Models::Cron.count} crons."
       Models::Cron.all.select do |cron|
-        @cron = cron
-        ne = cron.next_enqueuing
-        puts "Next enqueuing: #{ne}, time now: #{Time.now}, will it run? #{ne <= Time.now}"
-        start(cron) if ne <= Time.now
+        begin
+          @cron = cron
+          ne = cron.next_enqueuing
+          puts "Next enqueuing: #{ne}, time now: #{Time.now}, will it run? #{ne <= Time.now}"
+          start(cron) if ne <= Time.now
+        rescue => e
+          Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
+          sleep(10)
+          next
+        end
       end
-      rescue => e
-        puts "bad things happened"
-        puts e.inspect
-        puts Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
-        sleep(10)
     end
 
     def start(cron)
diff --git a/spec/v3/queries/cron_spec.rb b/spec/v3/queries/cron_spec.rb
index 7013495b..1418a6e0 100644
--- a/spec/v3/queries/cron_spec.rb
+++ b/spec/v3/queries/cron_spec.rb
@@ -4,6 +4,7 @@ describe Travis::API::V3::Queries::Crons do
   let(:user) { Travis::API::V3::Models::User.find_by_login('svenfuchs') }
   let(:repo) { Travis::API::V3::Models::Repository.where(owner_name: 'svenfuchs', name: 'minimal').first }
   let(:existing_branch) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron-test-existing', exists_on_github: true) }
+  let(:existing_branch2) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron-test-existing2', exists_on_github: true) }
   let(:non_existing_branch) { Travis::API::V3::Models::Branch.create(repository: repo, name: 'cron-test-non-existing', exists_on_github: false) }
   let(:query) { Travis::API::V3::Queries::Crons.new({}, 'Overview')
 }
@@ -25,9 +26,23 @@ describe Travis::API::V3::Queries::Crons do
     it 'enques error into a thread' do
       cron = Travis::API::V3::Models::Cron.create(branch_id: existing_branch.id, interval: 'daily', disable_by_build: false)
       error = StandardError.new('Konstantin broke all the thingz!')
+      Travis::API::V3::Queries::Crons.any_instance.expects(:sleep).with(10)
       Travis::API::V3::Models::Cron.any_instance.stubs(:branch).raises(error)
       Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron.id })
       query.start_all
     end
+
+    it 'continues running crons if one breaks' do
+      cron = Travis::API::V3::Models::Cron.create(branch_id: existing_branch.id, interval: 'daily', disable_by_build: false)
+      cron2 = Travis::API::V3::Models::Cron.create(branch_id: existing_branch2.id, interval: 'daily', disable_by_build: false)
+      
+      error = StandardError.new('Konstantin broke all the thingz!')
+      Travis::API::V3::Models::Cron.any_instance.stubs(:branch).raises(error)
+
+      Travis::API::V3::Queries::Crons.any_instance.expects(:sleep).twice.with(10)
+      Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron.id })
+      Raven.expects(:capture_exception).with(error, tags: {'cron_id' => cron2.id })
+      query.start_all
+    end
   end
 end

From 77e47bc41af2d31727acce23a440f41a1d3fa930 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Wed, 3 Aug 2016 00:13:43 -0400
Subject: [PATCH 10/11] don't report console errors to Sentry

---
 lib/travis/api/app.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/travis/api/app.rb b/lib/travis/api/app.rb
index fe1e8e7b..d61a498b 100644
--- a/lib/travis/api/app.rb
+++ b/lib/travis/api/app.rb
@@ -195,7 +195,7 @@ module Travis::Api
           end
         end
 
-        if use_monitoring? #&& !console?
+        if use_monitoring? && !console?
           setup_monitoring
         end
       end

From 8655579a94acdc8cb9ecdd31ad72fd427ba9ba67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= <renee@travis-ci.org>
Date: Wed, 3 Aug 2016 00:17:11 -0400
Subject: [PATCH 11/11] clean up the debug output

---
 lib/travis/api/v3/queries/crons.rb | 7 ++-----
 script/start_crons                 | 4 ----
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/lib/travis/api/v3/queries/crons.rb b/lib/travis/api/v3/queries/crons.rb
index f848b6e3..0932d178 100644
--- a/lib/travis/api/v3/queries/crons.rb
+++ b/lib/travis/api/v3/queries/crons.rb
@@ -6,16 +6,13 @@ module Travis::API::V3
     end
 
     def start_all()
-      puts "reviewing #{Models::Cron.count} crons."
       Models::Cron.all.select do |cron|
         begin
           @cron = cron
-          ne = cron.next_enqueuing
-          puts "Next enqueuing: #{ne}, time now: #{Time.now}, will it run? #{ne <= Time.now}"
-          start(cron) if ne <= Time.now
+          start(cron) if cron.next_enqueuing <= Time.now
         rescue => e
           Raven.capture_exception(e, tags: { 'cron_id' => @cron.try(:id) })
-          sleep(10)
+          sleep(10) # This ensures the dyno does not spin down before the http request to send the error to sentry completes
           next
         end
       end
diff --git a/script/start_crons b/script/start_crons
index 13e69b46..63a0273e 100755
--- a/script/start_crons
+++ b/script/start_crons
@@ -6,8 +6,4 @@ require 'travis/api/app'
 
 Travis::Api::App.setup
 
-puts "start"
-
 Travis::API::V3::Services::Crons::Start.new(nil,nil,nil).run!
-
-puts "done"