diff --git a/travis-api.gemspec b/travis-api.gemspec index 6bc91aa6..e6d94f61 100644 --- a/travis-api.gemspec +++ b/travis-api.gemspec @@ -598,7 +598,6 @@ Gem::Specification.new do |s| ======= "travis-api.gemspec", "vendor/travis-core/lib/travis.rb", - "vendor/travis-core/lib/travis/README.markdown", "vendor/travis-core/lib/travis/addons.rb", "vendor/travis-core/lib/travis/addons/README.markdown", "vendor/travis-core/lib/travis/addons/archive.rb", @@ -696,16 +695,9 @@ Gem::Specification.new do |s| "vendor/travis-core/lib/travis/api/v1/webhook/build/finished.rb", "vendor/travis-core/lib/travis/api/v1/webhook/build/finished/job.rb", "vendor/travis-core/lib/travis/api/v2.rb", - "vendor/travis-core/lib/travis/chunkifier.rb", - "vendor/travis-core/lib/travis/commit_command.rb", "vendor/travis-core/lib/travis/config/database.rb", "vendor/travis-core/lib/travis/config/defaults.rb", "vendor/travis-core/lib/travis/config/url.rb", - "vendor/travis-core/lib/travis/engine.rb", - "vendor/travis-core/lib/travis/enqueue.rb", - "vendor/travis-core/lib/travis/enqueue/services.rb", - "vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb", - "vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb", "vendor/travis-core/lib/travis/errors.rb", "vendor/travis-core/lib/travis/event.rb", "vendor/travis-core/lib/travis/event/config.rb", @@ -717,7 +709,6 @@ Gem::Specification.new do |s| "vendor/travis-core/lib/travis/github.rb", "vendor/travis-core/lib/travis/github/education.rb", "vendor/travis-core/lib/travis/github/services.rb", - "vendor/travis-core/lib/travis/github/services/fetch_config.rb", "vendor/travis-core/lib/travis/github/services/find_or_create_org.rb", "vendor/travis-core/lib/travis/github/services/find_or_create_repo.rb", "vendor/travis-core/lib/travis/github/services/find_or_create_user.rb", @@ -728,11 +719,6 @@ Gem::Specification.new do |s| "vendor/travis-core/lib/travis/github/services/sync_user/repository.rb", "vendor/travis-core/lib/travis/github/services/sync_user/reset_token.rb", "vendor/travis-core/lib/travis/github/services/sync_user/user_info.rb", - "vendor/travis-core/lib/travis/logs.rb", - "vendor/travis-core/lib/travis/logs/services.rb", - "vendor/travis-core/lib/travis/logs/services/aggregate.rb", - "vendor/travis-core/lib/travis/logs/services/archive.rb", - "vendor/travis-core/lib/travis/logs/services/receive.rb", "vendor/travis-core/lib/travis/mailer.rb", "vendor/travis-core/lib/travis/mailer/user_mailer.rb", "vendor/travis-core/lib/travis/mailer/views/layouts/contact_email.html.erb", @@ -798,13 +784,6 @@ Gem::Specification.new do |s| "vendor/travis-core/lib/travis/notification/publisher/redis.rb", "vendor/travis-core/lib/travis/overwritable_method_definitions.rb", "vendor/travis-core/lib/travis/redis_pool.rb", - "vendor/travis-core/lib/travis/requests.rb", - "vendor/travis-core/lib/travis/requests/services.rb", - "vendor/travis-core/lib/travis/requests/services/receive.rb", - "vendor/travis-core/lib/travis/requests/services/receive/api.rb", - "vendor/travis-core/lib/travis/requests/services/receive/cron.rb", - "vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb", - "vendor/travis-core/lib/travis/requests/services/receive/push.rb", "vendor/travis-core/lib/travis/secure_config.rb", "vendor/travis-core/lib/travis/services.rb", "vendor/travis-core/lib/travis/services/base.rb", @@ -859,7 +838,6 @@ Gem::Specification.new do |s| "vendor/travis-core/lib/travis/testing/scenario.rb", "vendor/travis-core/lib/travis/testing/stubs.rb", "vendor/travis-core/lib/travis/testing/stubs/stub.rb", - "vendor/travis-core/lib/travis/travis_yml_stats.rb", "vendor/travis-core/lib/travis_core/version.rb", "vendor/travis-core/travis-core.gemspec" >>>>>>> renamed the core spec directory diff --git a/vendor/travis-core/lib/travis.rb b/vendor/travis-core/lib/travis.rb index 6bb017cd..87df1658 100644 --- a/vendor/travis-core/lib/travis.rb +++ b/vendor/travis-core/lib/travis.rb @@ -43,14 +43,10 @@ module Travis require 'travis/addons' require 'travis/api' require 'travis/config/defaults' - require 'travis/commit_command' - require 'travis/enqueue' require 'travis/features' require 'travis/github' - require 'travis/logs' require 'travis/mailer' require 'travis/notification' - require 'travis/requests' require 'travis/services' class UnknownRepository < StandardError; end @@ -71,10 +67,7 @@ module Travis Github.setup Addons.register Services.register - Enqueue::Services.register Github::Services.register - Logs::Services.register - Requests::Services.register end attr_accessor :redis, :config diff --git a/vendor/travis-core/lib/travis/README.markdown b/vendor/travis-core/lib/travis/README.markdown deleted file mode 100644 index a24a9c90..00000000 --- a/vendor/travis-core/lib/travis/README.markdown +++ /dev/null @@ -1,16 +0,0 @@ -# Travis Core directory overview - -This folder, `lib/travis` contains the main code for the Travis Core repository. It contains several sub-section/subdirectories: - -- [`addons`](addons): Event handlers that take events such as "build finished" and sends out notifications to GitHub, Pusher, Campfire, etc. -- [`api`](api): Serializers for models and events used in our API and in some other places (for example to generate Pusher payloads). -- [`enqueue`](enqueue): Logic for enqueueing jobs. -- [`event`](event): Code for sending and subscribing to events. Used by the `addons` code to subscribe to changes in the models. -- [`github`](github): Services for communicating with the GitHub API. -- [`mailer`](mailer): ActionMailer mailers. -- [`model`](model): All of our ActiveRecord models. -- [`notification`](notification): Code for adding instrumentation. -- [`requests`](requests): Handles requests received from GitHub. -- [`secure_config.rb`](secure_config.rb): Logic for encrypting and decrypting build/job configs. -- [`services`](services): Most of the business logic behind our [API](https://github.com/travis-ci/travis-api). -- [`testing`](testing): Code used by our tests, such as model stubs and factories. diff --git a/vendor/travis-core/lib/travis/advisory_locks.rb b/vendor/travis-core/lib/travis/advisory_locks.rb deleted file mode 100644 index 4edae945..00000000 --- a/vendor/travis-core/lib/travis/advisory_locks.rb +++ /dev/null @@ -1,59 +0,0 @@ -require 'zlib' - -module Travis - # http://hashrocket.com/blog/posts/advisory-locks-in-postgres - # https://github.com/mceachen/with_advisory_lock - # 13.3.4. Advisory Locks : http://www.postgresql.org/docs/9.3/static/explicit-locking.html - # http://www.postgresql.org/docs/9.3/static/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS - class AdvisoryLocks - attr_reader :lock_name, :transactional - - def initialize(lock_name, transactional = false) - if lock_name.blank? - raise StandardError, "lock name cannot be blank" - end - @lock_name = lock_name - end - - # must be used within a transaction - def self.exclusive(lock_name, timeout = 30, transactional = false) - al = self.new(lock_name, transactional) - al.exclusive(timeout) { yield } - end - - # must be used within a transaction - def exclusive(timeout = 30) - give_up_at = Time.now + timeout if timeout - while timeout.nil? || Time.now < give_up_at do - if obtained_lock? - return yield - else - # Randomizing sleep time may help reduce contention. - sleep(rand(0.1..0.2)) - end - end - ensure - release_lock unless transactional - end - - private - - def obtained_lock? - xact = transactional ? "_xact" : nil - result = connection.select_value("select pg_try_advisory#{xact}_lock(#{lock_code});") - result == 't' || result == 'true' - end - - def release_lock - connection.execute("select pg_advisory_unlock(#{lock_code});") - end - - def connection - ActiveRecord::Base.connection - end - - def lock_code - Zlib.crc32(lock_name) - end - end -end \ No newline at end of file diff --git a/vendor/travis-core/lib/travis/chunkifier.rb b/vendor/travis-core/lib/travis/chunkifier.rb deleted file mode 100644 index 39176887..00000000 --- a/vendor/travis-core/lib/travis/chunkifier.rb +++ /dev/null @@ -1,59 +0,0 @@ -require 'coder/cleaner/simple/encodings' - -module Travis - class Chunkifier < Struct.new(:content, :chunk_size, :options) - include Enumerable - include Coder::Cleaner::Simple::Encodings::UTF_8 - - def initialize(*) - super - - self.options ||= {} - end - - def json? - options[:json] - end - - def length - parts.length - end - - def each(&block) - parts.each(&block) - end - - def parts - @parts ||= split - end - - def split - parts = content.scan(/.{1,#{chunk_split_size}}/m) - chunks = [] - current_chunk = '' - - parts.each do |part| - if too_big?(current_chunk + part) - chunks << current_chunk - current_chunk = part - else - current_chunk << part - end - end - - chunks << current_chunk if current_chunk.length > 0 - - chunks - end - - def chunk_split_size - size = chunk_size / 10 - size == 0 ? 1 : size - end - - def too_big?(current_chunk) - current_chunk = current_chunk.to_json if json? - current_chunk.bytesize > chunk_size - end - end -end diff --git a/vendor/travis-core/lib/travis/commit_command.rb b/vendor/travis-core/lib/travis/commit_command.rb deleted file mode 100644 index 2331387f..00000000 --- a/vendor/travis-core/lib/travis/commit_command.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Travis - class CommitCommand - - def initialize(message) - @message = message.to_s - end - - def skip? - backwards_skip or command == 'skip' - end - - private - attr_reader :message - - def command - message =~ /\[ci(?: |:)([\w ]*)\]/i && $1.downcase - end - - def backwards_skip - message =~ /\[skip\s+ci\]/i && true - end - end -end diff --git a/vendor/travis-core/lib/travis/engine.rb b/vendor/travis-core/lib/travis/engine.rb deleted file mode 100644 index 875a4947..00000000 --- a/vendor/travis-core/lib/travis/engine.rb +++ /dev/null @@ -1,21 +0,0 @@ -require 'travis' -require 'rails/engine' - -module Travis - class Engine < Rails::Engine - initializer 'add migrations path' do |app| - # need to insert to both Rails.app.paths and Migrator.migration_paths - # because Rails' stupid rake tasks copy them over before loading the - # engines *unless* multiple rake db tasks are combined (as in rake - # db:create db:migrate). Happens in Rails <= 3.2.2 - paths = [ - Rails.application.paths['db/migrate'], - ActiveRecord::Migrator.migrations_paths - ] - paths.each do |paths| - path = root.join('db/migrate').to_s - paths << path unless paths.include?(path) - end - end - end -end diff --git a/vendor/travis-core/lib/travis/enqueue.rb b/vendor/travis-core/lib/travis/enqueue.rb deleted file mode 100644 index e5ac1d96..00000000 --- a/vendor/travis-core/lib/travis/enqueue.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Travis - module Enqueue - require 'travis/enqueue/services' - end -end - diff --git a/vendor/travis-core/lib/travis/enqueue/services.rb b/vendor/travis-core/lib/travis/enqueue/services.rb deleted file mode 100644 index 9228aff5..00000000 --- a/vendor/travis-core/lib/travis/enqueue/services.rb +++ /dev/null @@ -1,14 +0,0 @@ -module Travis - module Enqueue - module Services - require 'travis/enqueue/services/enqueue_jobs' - - class << self - def register - constants(false).each { |name| const_get(name) } - end - end - end - end -end - diff --git a/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb b/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb deleted file mode 100644 index 4e52275c..00000000 --- a/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb +++ /dev/null @@ -1,127 +0,0 @@ -require 'travis/services/base' -require 'travis/support/instrumentation' -require 'travis/support/exceptions/handling' - -module Travis - module Enqueue - module Services - # Finds owners that have queueable jobs and for each owner: - # - # * checks how many jobs can be enqueued - # * finds the oldest N queueable jobs and - # * enqueues them - class EnqueueJobs < Travis::Services::Base - TIMEOUT = 2 - - extend Travis::Instrumentation, Travis::Exceptions::Handling - - require 'travis/enqueue/services/enqueue_jobs/limit' - - register :enqueue_jobs - - def self.run - new.run - end - - def reports - @reports ||= {} - end - - def run - enqueue_all && reports unless disabled? - end - instrument :run - rescues :run, from: Exception, backtrace: false - - def disabled? - Timeout.timeout(TIMEOUT) do - Travis::Features.feature_deactivated?(:job_queueing) - end - rescue Timeout::Error, Redis::TimeoutError => e - Travis.logger.error("[enqueue] Timeout trying to check enqueuing feature flag.") - return false - end - - private - - def enqueue_all - grouped_jobs = jobs.group_by(&:owner) - - Metriks.timer('enqueue.total').time do - grouped_jobs.each do |owner, jobs| - next unless owner - Metriks.timer('enqueue.full_enqueue_per_owner').time do - limit = nil - queueable = nil - Metriks.timer('enqueue.limit_per_owner').time do - Travis.logger.info "About to evaluate jobs for: #{owner.login}." - limit = Limit.new(owner, jobs) - queueable = limit.queueable - end - - Metriks.timer('enqueue.enqueue_per_owner').time do - enqueue(queueable) - end - - Metriks.timer('enqueue.report_per_owner').time do - reports[owner.login] = limit.report - end - end - end - end - end - - def enqueue(jobs) - jobs.each do |job| - Travis.logger.info("enqueueing slug=#{job.repository.slug} job_id=#{job.id}") - Metriks.timer('enqueue.publish_job').time do - publish(job) - end - - Metriks.timer('enqueue.enqueue_job').time do - job.enqueue - end - end - end - - def publish(job) - Metriks.timer('enqueue.publish_job').time do - payload = Travis::Api.data(job, for: 'worker', type: 'Job::Test', version: 'v0') - publisher(job.queue).publish(payload, properties: { type: payload['type'], persistent: true }) - end - end - - def jobs - Metriks.timer('enqueue.fetch_jobs').time do - jobs = Job.includes(:owner).queueable.all - Travis.logger.info "Found #{jobs.size} jobs in total." if jobs.size > 0 - jobs - end - end - - def publisher(queue) - Travis::Amqp::Publisher.builds(queue) - end - - class Instrument < Notification::Instrument - def run_completed - publish(msg: format(target.reports), reports: target.reports) - end - - def format(reports) - reports = Array(reports) - if reports.any? - reports = reports.map do |repo, report| - " #{repo}: #{report.map { |key, value| "#{key}: #{value}" }.join(', ')}" - end - "enqueued:\n#{reports.join("\n")}" - else - 'nothing to enqueue.' - end - end - end - Instrument.attach_to(self) - end - end - end -end diff --git a/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb b/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb deleted file mode 100644 index 3d23ef2f..00000000 --- a/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb +++ /dev/null @@ -1,85 +0,0 @@ -require 'travis/services/base' -require 'travis/model/job' - -module Travis - module Enqueue - module Services - class EnqueueJobs < Travis::Services::Base - class Limit - attr_reader :owner, :jobs, :config - - def initialize(owner, jobs) - @owner = owner - @jobs = jobs - @config = Travis.config.queue.limit - end - - def queueable - @queueable ||= filter_by_repository(jobs)[0, max_queueable] - end - - def filter_by_repository(jobs) - return jobs unless Travis.config.limit_per_repo_enabled? - queueable_by_repository_id = {} - jobs.reject do |job| - if job.repository.settings.restricts_number_of_builds? - queueable?(job, queueable_by_repository_id, running_by_repository_id) - end - end - end - - def running_by_repository_id - @running_by_repository ||= Hash[running_jobs.group_by(&:repository_id).map {|repository_id, jobs| [repository_id, jobs.size]}] - end - - def queueable?(job, queueable, running) - repository = job.repository_id - queueable[repository] ||= 0 - - runnable_count = queueable[repository] + - (running[repository] || 0) - if runnable_count < job.repository.settings.maximum_number_of_builds - queueable[repository] += 1 - false - else - true - end - end - - def report - { total: jobs.size, running: running, max: max_jobs, queueable: queueable.size } - end - - private - - def running_jobs - @running_jobs ||= Job.owned_by(owner).running - end - - def running - @running ||= Job.owned_by(owner).running.count(:id) - end - - def max_queueable - return config.default if owner.login.nil? - - if unlimited? - 999 - else - queueable = max_jobs - running - queueable < 0 ? 0 : queueable - end - end - - def max_jobs - config.by_owner[owner.login] || config.default - end - - def unlimited? - config.by_owner[owner.login] == -1 - end - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/github/services.rb b/vendor/travis-core/lib/travis/github/services.rb index fa408c79..bca56e28 100644 --- a/vendor/travis-core/lib/travis/github/services.rb +++ b/vendor/travis-core/lib/travis/github/services.rb @@ -1,7 +1,6 @@ module Travis module Github module Services - require 'travis/github/services/fetch_config' require 'travis/github/services/find_or_create_org' require 'travis/github/services/find_or_create_repo' require 'travis/github/services/find_or_create_user' diff --git a/vendor/travis-core/lib/travis/github/services/fetch_config.rb b/vendor/travis-core/lib/travis/github/services/fetch_config.rb deleted file mode 100644 index 3d16aa15..00000000 --- a/vendor/travis-core/lib/travis/github/services/fetch_config.rb +++ /dev/null @@ -1,90 +0,0 @@ -require 'gh' -require 'yaml' -require 'active_support/core_ext/class/attribute' -require 'travis/support/logging' -require 'travis/support/instrumentation' -require 'travis/services/base' - -module Travis - module Github - module Services - # encapsulates fetching a .travis.yml from a given commit's config_url - class FetchConfig < Travis::Services::Base - include Logging - extend Instrumentation - - register :github_fetch_config - - def run - config = retrying(3) { filter(parse(fetch)) } - config || Travis.logger.warn("[request:fetch_config] Empty config for request id=#{request.id} config_url=#{config_url.inspect}") - rescue GH::Error => e - if e.info[:response_status] == 404 - { '.result' => 'not_found' } - else - { '.result' => 'server_error' } - end - end - instrument :run - - def request - params[:request] - end - - def config_url - request.config_url - end - - private - - def fetch - content = GH[config_url]['content'] - Travis.logger.warn("[request:fetch_config] Empty content for #{config_url}") if content.nil? - content = content.to_s.unpack('m').first - Travis.logger.warn("[request:fetch_config] Empty unpacked content for #{config_url}, content was #{content.inspect}") if content.nil? - nbsp = "\xC2\xA0".force_encoding("binary") - content = content.gsub(/^(#{nbsp})+/) { |match| match.gsub(nbsp, " ") } - - content - end - - def parse(yaml) - YAML.load(yaml).merge('.result' => 'configured') - rescue StandardError, Psych::SyntaxError => e - error "[request:fetch_config] Error parsing .travis.yml for #{config_url}: #{e.message}" - { - '.result' => 'parse_error', - '.result_message' => e.is_a?(Psych::SyntaxError) ? e.message.split(": ").last : e.message - } - end - - def filter(config) - unless Travis::Features.active?(:template_selection, request.repository) - config = config.to_h.except('dist').except('group') - end - - config - end - - def retrying(times) - count, result = 0, nil - until result || count > times - result = yield - count += 1 - Travis.logger.warn("[request:fetch_config] Retrying to fetch config for #{config_url}") unless result - end - result - end - - class Instrument < Notification::Instrument - def run_completed - # TODO exctract something like Url.strip_secrets - config_url = target.config_url.gsub(/(token|secret)=\w*/) { "#{$1}=[secure]" } - publish(msg: "#{config_url}", url: config_url, result: result) - end - end - Instrument.attach_to(self) - end - end - end -end diff --git a/vendor/travis-core/lib/travis/logs.rb b/vendor/travis-core/lib/travis/logs.rb deleted file mode 100644 index 89dfa53b..00000000 --- a/vendor/travis-core/lib/travis/logs.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Travis - module Logs - autoload :Services, 'travis/logs/services' - end -end - diff --git a/vendor/travis-core/lib/travis/logs/services.rb b/vendor/travis-core/lib/travis/logs/services.rb deleted file mode 100644 index f235cf5a..00000000 --- a/vendor/travis-core/lib/travis/logs/services.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Travis - module Logs - module Services - autoload :Aggregate, 'travis/logs/services/aggregate' - autoload :Archive, 'travis/logs/services/archive' - autoload :Receive, 'travis/logs/services/receive' - - class << self - def register - constants(false).each { |name| const_get(name) } - end - end - end - end -end - - diff --git a/vendor/travis-core/lib/travis/logs/services/aggregate.rb b/vendor/travis-core/lib/travis/logs/services/aggregate.rb deleted file mode 100644 index 944f2f9d..00000000 --- a/vendor/travis-core/lib/travis/logs/services/aggregate.rb +++ /dev/null @@ -1,96 +0,0 @@ -require 'active_support/core_ext/string/filters' -require 'travis/features' -require 'travis/model/log' -require 'travis/model/log/part' -require 'travis/services/base' - -module Travis - module Logs - module Services - class Aggregate < Travis::Services::Base - register :logs_aggregate - - AGGREGATE_UPDATE_SQL = <<-sql.squish - UPDATE logs - SET aggregated_at = ?, - content = (COALESCE(content, '') || (#{Log::AGGREGATE_PARTS_SELECT_SQL})) - WHERE logs.id = ? - sql - - AGGREGATEABLE_SELECT_SQL = <<-sql.squish - SELECT DISTINCT log_id - FROM log_parts - WHERE created_at <= NOW() - interval '? seconds' AND final = ? - OR created_at <= NOW() - interval '? seconds' - sql - - def run - return unless active? - aggregateable_ids.each do |id| - transaction do - aggregate(id) - vacuum(id) - notify(id) - end - end - end - - private - - def active? - Travis::Features.feature_active?(:log_aggregation) - end - - def aggregate(id) - meter('logs.aggregate') do - connection.execute(sanitize_sql([AGGREGATE_UPDATE_SQL, Time.now, id, id])) - end - end - - def vacuum(id) - meter('logs.vacuum') do - Log::Part.delete_all(log_id: id) - end - end - - def notify(id) - Log.find(id).notify('aggregated') - rescue ActiveRecord::RecordNotFound - puts "[warn] could not find a log with the id #{id}" - end - - def aggregateable_ids - Log::Part.connection.select_values(query).map { |id| id.nil? ? id : id.to_i } - end - - def query - Log::Part.send(:sanitize_sql, [AGGREGATEABLE_SELECT_SQL, intervals[:regular], true, intervals[:force]]) - end - - def intervals - Travis.config.logs.intervals - end - - def transaction(&block) - ActiveRecord::Base.transaction(&block) - rescue ActiveRecord::ActiveRecordError => e - # puts e.message, e.backtrace - Travis::Exceptions.handle(e) - end - - def meter(name, &block) - Metriks.timer(name).time(&block) - end - - def connection - Log::Part.connection - end - - def sanitize_sql(*args) - Log::Part.send(:sanitize_sql, *args) - end - end - end - end -end - diff --git a/vendor/travis-core/lib/travis/logs/services/archive.rb b/vendor/travis-core/lib/travis/logs/services/archive.rb deleted file mode 100644 index 98890b31..00000000 --- a/vendor/travis-core/lib/travis/logs/services/archive.rb +++ /dev/null @@ -1,167 +0,0 @@ -begin - require 'aws/s3' -rescue LoadError => e -end -require 'uri' -require 'active_support/core_ext/hash/slice' -require 'faraday' -require 'travis/support/instrumentation' -require 'travis/notification/instrument' -require 'travis/services/base' - -module Travis - class S3 - class << self - def setup - AWS.config(Travis.config.s3.to_h.slice(:access_key_id, :secret_access_key)) - end - end - - attr_reader :s3, :url - - def initialize(url) - @s3 = AWS::S3.new - @url = url - end - - def store(data) - object.write(data, content_type: 'text/plain', acl: :public_read) - end - - def object - @object ||= bucket.objects[URI.parse(url).path[1..-1]] - end - - def bucket - @bucket ||= s3.buckets[URI.parse(url).host] - end - end - - module Logs - module Services - class Archive < Travis::Services::Base - class FetchFailed < StandardError - def initialize(source_url, status, message) - super("Could not retrieve #{source_url}. Response status: #{status}, message: #{message}") - end - end - - class VerificationFailed < StandardError - def initialize(source_url, target_url, expected, actual) - super("Expected #{target_url} (from: #{source_url}) to have the content length #{expected.inspect}, but had #{actual.inspect}") - end - end - - extend Travis::Instrumentation - - register :archive_log - - attr_reader :log - - def run - fetch - store - verify - report - end - instrument :run - - def source_url - "https://#{hostname('api')}/logs/#{params[:id]}.txt" - end - - def report_url - "https://#{hostname('api')}/logs/#{params[:id]}" - end - - def target_url - "http://#{hostname('archive')}/jobs/#{params[:job_id]}/log.txt" - end - - private - - def fetch - retrying(:fetch) do - response = request(:get, source_url) - if response.status == 200 - @log = response.body.to_s - else - raise(FetchFailed.new(source_url, response.status, response.body.to_s)) - end - end - end - - def store - retrying(:store) do - S3.setup - s3.store(log) - end - end - - def verify - retrying(:verify) do - expected = log.bytesize - actual = request(:head, target_url).headers['content-length'].try(:to_i) - raise VerificationFailed.new(target_url, source_url, expected, actual) unless expected == actual - end - end - - def report - retrying(:report) do - request(:put, report_url, { archived_at: Time.now, archive_verified: true }, token: Travis.config.tokens.internal) - end - end - - def request(method, url, params = nil, headers = nil, &block) - http.send(*[method, url, params, headers].compact, &block) - rescue Faraday::Error => e - puts "Exception while trying to #{method.inspect}: #{source_url}:" - puts e.message, e.backtrace - raise e - end - - def http - Faraday.new(ssl: Travis.config.ssl.to_h.compact) do |f| - f.request :url_encoded - f.adapter :net_http - end - end - - def s3 - S3.new(target_url) - end - - def hostname(name) - "#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}" - end - - def retrying(header, times = 5) - yield - rescue => e - count ||= 0 - if times > (count += 1) - puts "[#{header}] retry #{count} because: #{e.message}" - Travis::Instrumentation.meter("#{self.class.name.underscore.gsub("/", ".")}.retries.#{header}") - sleep count * 3 unless params[:no_sleep] - retry - else - raise - end - end - - class Instrument < Notification::Instrument - def run_completed - publish( - msg: "for (to: #{target.target_url})", - source_url: target.source_url, - target_url: target.target_url, - object_type: 'Log', - object_id: target.params[:id] - ) - end - end - Instrument.attach_to(self) - end - end - end -end diff --git a/vendor/travis-core/lib/travis/logs/services/receive.rb b/vendor/travis-core/lib/travis/logs/services/receive.rb deleted file mode 100644 index 8f56cc80..00000000 --- a/vendor/travis-core/lib/travis/logs/services/receive.rb +++ /dev/null @@ -1,79 +0,0 @@ -require 'coder' -require 'metriks' -require 'travis/model/log/part' -require 'travis/model/job/test' -require 'travis/services/base' - -module Travis - module Logs - module Services - class Receive < Travis::Services::Base - # TODO remove this once we know aggregation works fine and the worker passes a :final flag - FINAL = 'Done. Build script exited with:' - - register :logs_receive - - def run - create_part - notify - end - - private - - def create_part - measure('logs.update') do - Travis.logger.warn "[warn] log.id is #{log.id.inspect} in :logs_append! job_id: #{data[:id]}" if log.id.to_i == 0 - Log::Part.create!(log_id: log.id, content: chars, number: number, final: final?) - end - rescue ActiveRecord::ActiveRecordError => e - Travis.logger.warn "[warn] could not save log in :logs_append job_id: #{data[:id]}: #{e.message}" - Travis.logger.warn e.backtrace - end - - def notify - job.notify(:log, _log: chars, number: number, final: final?) - rescue => e - Metriks.meter('travis.logs.update.notify.errors').mark - Travis.logger.error("Error notifying of log update: #{e.message} (from #{e.backtrace.first})") - end - - def log - @log ||= Log.where(job_id: job.id).select(:id).first || create_log - end - - def create_log - Travis.logger.warn "[warn] Had to create a log for job_id: #{job.id}!" - job.create_log! - end - - def job - @job ||= Job::Test.find(data[:id]) - end - - def chars - @chars ||= filter(data[:log]) - end - - def number - data[:number] - end - - def final? - !!data[:final] || chars.include?(FINAL) - end - - def data - @data ||= params[:data].symbolize_keys - end - - def filter(chars) - Coder.clean!(chars.to_s.gsub("\0", '')) # postgres seems to have issues with null chars - end - - def measure(name, &block) - Metriks.timer(name).time(&block) - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/requests.rb b/vendor/travis-core/lib/travis/requests.rb deleted file mode 100644 index 2ff1043b..00000000 --- a/vendor/travis-core/lib/travis/requests.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Travis - module Requests - require 'travis/requests/services' - end -end - diff --git a/vendor/travis-core/lib/travis/requests/services.rb b/vendor/travis-core/lib/travis/requests/services.rb deleted file mode 100644 index 8ea5f674..00000000 --- a/vendor/travis-core/lib/travis/requests/services.rb +++ /dev/null @@ -1,13 +0,0 @@ -module Travis - module Requests - module Services - require 'travis/requests/services/receive' - - class << self - def register - constants(false).each { |name| const_get(name) } - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/requests/services/receive.rb b/vendor/travis-core/lib/travis/requests/services/receive.rb deleted file mode 100644 index b0f55dab..00000000 --- a/vendor/travis-core/lib/travis/requests/services/receive.rb +++ /dev/null @@ -1,207 +0,0 @@ -require 'gh' -require 'travis/services/base' -require 'travis/model/request/approval' -require 'travis/notification/instrument' -require 'travis/advisory_locks' -require 'travis/travis_yml_stats' - -module Travis - module Requests - module Services - class Receive < Travis::Services::Base - require 'travis/requests/services/receive/api' - require 'travis/requests/services/receive/cron' - require 'travis/requests/services/receive/pull_request' - require 'travis/requests/services/receive/push' - - extend Travis::Instrumentation - - class PayloadValidationError < StandardError; end - - register :receive_request - - class << self - def payload_for(type, data) - data = GH.load(data) - const_get(type.camelize).new(data) - end - end - - attr_reader :request, :accepted - - def run - with_transactional_advisory_lock do - if accept? - create && start - store_config_info if verify - else - rejected - end - request - end - rescue GH::Error => e - Travis.logger.error "payload for #{slug} could not be received as GitHub returned a #{e.info[:response_status]}: #{e.info}, github-guid=#{github_guid}, event-type=#{event_type}" - end - instrument :run - - def accept? - payload.validate! - validate! - @accepted = payload.accept? - rescue PayloadValidationError => e - Travis.logger.error "#{e.message}, github-guid=#{github_guid}, event-type=#{event_type}" - @accepted = false - end - - private - - def with_transactional_advisory_lock - return yield unless payload.repository - result = nil - Travis::AdvisoryLocks.exclusive("receive-repo:#{payload.repository[:github_id]}", 300) do - ActiveRecord::Base.connection.transaction do - result = yield - end - end - result - rescue => e - ActiveRecord::Base.connection.rollback_db_transaction - raise - end - - def validate! - repo_not_found! unless repo - verify_owner - end - - def verify_owner - owner = owner_by_payload - owner_not_found! unless owner - update_owner(owner) if owner.id != repo.owner_id && !api_request? - end - - def create - @request = repo.requests.create!(payload.request.merge( - :payload => params[:payload], - :event_type => event_type, - :state => :created, - :commit => commit, - :owner => repo.owner, - :token => params[:token] - )) - end - - def start - request.start! - end - - def verify - request.reload - if request.builds.count == 0 - approval = Request::Approval.new(request) - Travis.logger.warn("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} didn't create any builds: #{approval.result}/#{approval.message}") - false - elsif !request.creates_jobs? - approval = Request::Approval.new(request) - Travis.logger.warn("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} didn't create any job: #{approval.result}/#{approval.message}") - false - else - Travis.logger.info("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} created #{request.builds.count} builds") - true - end - end - - def update_owner(owner) - repo.update_attributes!(owner: owner, owner_name: owner.login) - owner_updated - end - - def owner_by_payload - if id = payload.repository[:owner_id] - lookup_owner(payload.repository[:owner_type], id: id) - elsif github_id = payload.repository[:owner_github_id] - lookup_owner(payload.repository[:owner_type], github_id: github_id) - elsif login = payload.repository[:owner_name] - lookup_owner(%w(User Organization), login: login) - end - end - - def lookup_owner(types, attrs) - Array(types).map(&:constantize).each do |type| - owner = type.where(attrs).first - return owner if owner - end - nil - end - - def repo_not_found! - Travis::Metrics.meter('request.receive.repository_not_found') - raise PayloadValidationError, "Repository not found: #{payload.repository.slice(:id, :github_id, :owner_name, :name)}" - end - - def owner_not_found! - Travis::Metrics.meter('request.receive.repository_owner_not_found') - raise PayloadValidationError, "The given repository owner could not be found: #{payload.repository.slice(:owner_id, :owner_github_id, :owner_type, :owner_name).inspect}" - end - - def owner_updated - Travis::Metrics.meter('request.receive.update_owner') - Travis.logger.warn("[request:receive] Repository owner updated for #{slug}: #{repo.owner_type}##{repo.owner_id} (#{repo.owner_name})") - end - - def rejected - commit = payload.commit['commit'].inspect if payload.commit rescue nil - Travis.logger.info("[request:receive] Github event rejected: event_type=#{event_type.inspect} repo=\"#{slug}\" commit=#{commit} action=#{payload.action.inspect}") - end - - def payload - @payload ||= self.class.payload_for(event_type, params[:payload]) - end - - def github_guid - params[:github_guid] - end - - def event_type - @event_type ||= (params[:event_type] || 'push').gsub('-', '_') - end - - def api_request? - event_type == 'api' - end - - def repo - @repo ||= run_service(:find_repo, payload.repository) - end - - def slug - payload.repository ? payload.repository.values_at(:owner_name, :name).join('/') : '?' - end - - def commit - @commit ||= repo.commits.create!(payload.commit) if payload.commit - end - - def store_config_info - Travis::TravisYmlStats.store_stats(request) - rescue => e - Travis.logger.warn("[request:receive] Couldn't store .travis.yml stats: #{e.message}") - Travis::Exceptions.handle(e) - end - - class Instrument < Notification::Instrument - def run_completed - params = target.params - publish( - :msg => "type=#{params[:event_type].inspect}", - :type => params[:event_type], - :accept? => target.accepted, - :payload => params[:payload] - ) - end - end - Instrument.attach_to(self) - end - end - end -end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/api.rb b/vendor/travis-core/lib/travis/requests/services/receive/api.rb deleted file mode 100644 index 1711b622..00000000 --- a/vendor/travis-core/lib/travis/requests/services/receive/api.rb +++ /dev/null @@ -1,106 +0,0 @@ -module Travis - module Requests - module Services - class Receive < Travis::Services::Base - class Api - VALIDATION_ERRORS = { - repo: 'Repository data is not present in payload', - } - attr_reader :event - - def initialize(event) - @event = event - end - - def accept? - true - end - - def validate! - error(:repo) if repo_data.nil? - end - - def action - nil - end - - def repository - @repository ||= { - owner_id: repo_data['owner_id'], - owner_type: repo_data['owner_type'], - owner_name: repo_data['owner_name'], - name: repo_data['name'] - } - end - - def request - @request ||= { - :config => event['config'] - } - end - - def commit - @commit ||= { - commit: commit_data['sha'], - message: message, - branch: branch, - ref: nil, # TODO verify that we do not need this - committed_at: commit_data['commit']['committer']['date'], # TODO in case of API requests we'd want to display the timestamp of the incoming request - committer_name: commit_data['commit']['committer']['name'], - committer_email: commit_data['commit']['committer']['email'], - author_name: commit_data['commit']['author']['name'], - author_email: commit_data['commit']['author']['email'], - compare_url: commit_data['_links']['self']['href'] - } - end - - private - - def gh - Github.authenticated(user) - end - - def user - @user ||= User.find(event['user']['id']) - end - - def repo_data - event['repository'] || {} - end - - def message - event['message'] || commit_data['commit']['message'] - end - - def slug - repo_data.values_at('owner_name', 'name').join('/') - end - - def branch - event['branch'] || 'master' - end - - def repo_github_id - repo.try(:github_id) || raise(ActiveRecord::RecordNotFound) - end - - def repo - if id = repo_data['id'] - Repository.find(id) - else - Repository.by_slug(slug).first - end - end - - def commit_data - @commit_data ||= gh["repos/#{slug}/commits?sha=#{branch}&per_page=1"].first # TODO I guess Api would protect against GH errors? - end - - def error(type) - raise PayloadValidationError, VALIDATION_ERRORS[type] - end - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/cron.rb b/vendor/travis-core/lib/travis/requests/services/receive/cron.rb deleted file mode 100644 index dfc3c743..00000000 --- a/vendor/travis-core/lib/travis/requests/services/receive/cron.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Travis - module Requests - module Services - class Receive < Travis::Services::Base - class Cron < Api - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb b/vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb deleted file mode 100644 index d09c5c0c..00000000 --- a/vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb +++ /dev/null @@ -1,117 +0,0 @@ -module Travis - module Requests - module Services - class Receive < Travis::Services::Base - class PullRequest - attr_reader :event - - def initialize(event) - @event = event - end - - def accept? - return false if disabled? || closed? - case action - when :opened, :reopened then !!merge_commit - when :synchronize then head_change? - else false - end - end - - def validate! - if event['repository'].nil? - raise PayloadValidationError, "Repository data is not present in payload" - end - end - - def disabled? - Travis::Features.feature_deactivated?(:pull_requests) - end - - def closed? - pull_request['state'] == 'closed' - end - - def head_change? - head_commit && ::Request.last_by_head_commit(head_commit['sha']).nil? - end - - def repository - @repository ||= repo && { - name: repo['name'], - description: repo['description'], - url: repo['_links']['html']['href'], - owner_github_id: repo['owner']['id'], - owner_type: repo['owner']['type'], - owner_name: repo['owner']['login'], - owner_email: repo['owner']['email'], - private: !!repo['private'], - github_id: repo['id'] - } - end - - def request - @request ||= { - comments_url: comments_url, - base_commit: base_commit['sha'], - head_commit: head_commit['sha'] - } - end - - def commit - @commit ||= if merge_commit - { - commit: merge_commit['sha'], - message: head_commit['message'], - branch: pull_request['base']['ref'], - ref: merge_commit['ref'], - committed_at: committer['date'], - committer_name: committer['name'], - committer_email: committer['email'], - author_name: author['name'], - author_email: author['email'], - compare_url: pull_request['_links']['html']['href'] - } - end - end - - def pull_request - event['pull_request'] || {} - end - - def action - event['action'].try(:to_sym) - end - - def comments_url - pull_request.fetch('_links', {}).fetch('comments', {}).fetch('href', '') - end - - def base_commit - pull_request['base_commit'] || { 'sha' => '' } - end - - def head_commit - pull_request['head_commit'] - end - - def merge_commit - pull_request['merge_commit'] - end - - def repo - event['repository'] - end - - def committer - head_commit.fetch('committer', {}) - end - - def author - head_commit.fetch('author', {}) - end - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/push.rb b/vendor/travis-core/lib/travis/requests/services/receive/push.rb deleted file mode 100644 index bab6b2df..00000000 --- a/vendor/travis-core/lib/travis/requests/services/receive/push.rb +++ /dev/null @@ -1,84 +0,0 @@ -module Travis - module Requests - module Services - class Receive < Travis::Services::Base - class Push - attr_reader :event - - def initialize(event) - @event = event - end - - def accept? - true - end - - def validate! - if event['repository'].nil? - raise PayloadValidationError, "Repository data is not present in payload" - end - end - - def action - nil - end - - def repository - @repository ||= repo_data && { - name: repo_data['name'], - description: repo_data['description'], - url: repo_data['_links']['html']['href'], - owner_github_id: repo_data['owner']['id'], - owner_type: repo_data['owner']['type'], - owner_name: repo_data['owner']['login'], - owner_email: repo_data['owner']['email'], - private: !!repo_data['private'], - github_id: repo_data['id'] - } - end - - def request - @request ||= {} - end - - def commit - @commit ||= commit_data && { - commit: commit_data['sha'], - message: commit_data['message'], - branch: event['ref'].split('/', 3).last, - ref: event['ref'], - committed_at: commit_data['date'], - committer_name: commit_data['committer']['name'], - committer_email: commit_data['committer']['email'], - author_name: commit_data['author']['name'], - author_email: commit_data['author']['email'], - compare_url: event['compare'] - } - end - - private - - def repo_data - event['repository'] - end - - def commit_data - last_unskipped_commit || commits.last || event['head_commit'] - end - - def last_unskipped_commit - commits.reverse.find { |commit| !skip_commit?(commit) } - end - - def commits - event['commits'] || [] - end - - def skip_commit?(commit) - Travis::CommitCommand.new(commit['message']).skip? - end - end - end - end - end -end diff --git a/vendor/travis-core/lib/travis/travis_yml_stats.rb b/vendor/travis-core/lib/travis/travis_yml_stats.rb deleted file mode 100644 index 776e9318..00000000 --- a/vendor/travis-core/lib/travis/travis_yml_stats.rb +++ /dev/null @@ -1,195 +0,0 @@ -require "sidekiq" - -begin - require "keen" -rescue LoadError -end - -module Travis - class TravisYmlStats - class KeenPublisher - include ::Sidekiq::Worker - - sidekiq_options queue: :keen_events - - def perform(payload, deployment_payload = nil, notification_payload = nil) - if defined?(Keen) && ENV["KEEN_PROJECT_ID"] - payload = { :requests => [payload] } - payload[:deployments] = deployment_payload if deployment_payload.to_a.size > 0 - payload[:notifications] = notification_payload if notification_payload.to_a.size > 0 - Keen.publish_batch(payload) - end - end - end - - LANGUAGE_VERSION_KEYS = %w[ - ghc - go - jdk - node_js - otp_release - perl - php - python - ruby - rvm - scala - ] - - def self.store_stats(request, publisher=KeenPublisher) - new(request, publisher).store_stats - end - - def initialize(request, publisher) - @request = request - @publisher = publisher - @keen_payload = {} - @keen_payload_deployment = [] - @keen_payload_notification = [] - end - - def store_stats - set_basic_info - set_language - set_language_version - set_uses_sudo - set_uses_apt_get - set_dist - set_group - set_deployment_provider_count - set_notification - - @publisher.perform_async(keen_payload, keen_payload_deployment, keen_payload_notification) - end - - private - - attr_reader :request, :keen_payload - attr_accessor :keen_payload_deployment - attr_accessor :keen_payload_notification - - def set(path, value, collection = keen_payload) - path = Array(path) - hsh = collection - path[0..-2].each do |key| - hsh[key.to_sym] ||= {} - hsh = hsh[key.to_sym] - end - - hsh[path.last.to_sym] = value - end - - def set_basic_info - set :event_type, request.event_type - set :matrix_size, request.builds.map { |build| build.matrix.size }.reduce(:+) - set :repository_id, request.repository_id - set :owner_id, request.owner_id - set :owner_type, request.owner_type - # The owner_type, owner_id tuple is there so we can do unique counts on it - set :owner, [request.owner_type, request.owner_id] - end - - def set_language - set :language, travis_yml_language - set :github_language, github_language - end - - def set_language_version - LANGUAGE_VERSION_KEYS.each do |key| - if config.key?(key) - case config[key] - when String, Array - set [:language_version, key], Array(config[key]).map(&:to_s).sort - else - set [:language_version, key], ["invalid"] - end - end - end - end - - def set_uses_sudo - set :uses_sudo, commands.any? { |command| command =~ /\bsudo\b/ } - end - - def set_uses_apt_get - set :uses_apt_get, commands.any? { |command| command =~ /\bapt-get\b/ } - end - - def set_dist - set :dist_name, dist_name - end - - def set_group - set :group_name, group_name - end - - def set_deployment_provider_count - deploy = config["deploy"] || return - # Hash#to_a is not what we want here - deployments = deploy.is_a?(Hash) ? [deploy] : Array(deploy) - deployments.map {|d| d["provider"] }.uniq.each do |provider| - keen_payload_deployment << { provider: provider.downcase, repository_id: request.repository_id } - end - rescue - nil - end - - def set_notification - notifications = config["notifications"] || return - notifications.keys.each do |notifier| - keen_payload_notification << { notifier: notifier.downcase, repository_id: request.repository_id } - end - rescue - nil - end - - def config - request.config - end - - def payload - request.payload.is_a?(String) ? MultiJson.decode(request.payload) : request.payload - end - - def commands - [ - config["before_install"], - config["install"], - config["before_script"], - config["script"], - config["after_success"], - config["after_failure"], - config["before_deploy"], - config["after_deploy"], - ].flatten.compact - end - - def travis_yml_language - language = config["language"] - case language - when String - language - when nil - "default" - else - "invalid" - end - end - - def github_language - payload.fetch("repository", {})["language"] - end - - def normalize_string(str) - str.downcase.gsub("#", "-sharp").gsub(/[^A-Za-z0-9.:\-_]/, "") - end - - def dist_name - config.fetch('dist', 'default') - end - - def group_name - config.fetch('group', 'default') - end - end -end