suzanne.soy/travis-api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

delete unused stuffz

Browse Source
This commit is contained in:
Sven Fuchs 2016-06-08 17:24:21 +02:00 committed by Tyranja
parent 0077b6414a
commit feb4bcd90f
26 changed files with 0 additions and 1633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
travis-api.gemspec
View File

@ -598,7 +598,6 @@ Gem::Specification.new do |s|
=======
"travis-api.gemspec",
"vendor/travis-core/lib/travis.rb",
"vendor/travis-core/lib/travis/README.markdown",
"vendor/travis-core/lib/travis/addons.rb",
"vendor/travis-core/lib/travis/addons/README.markdown",
"vendor/travis-core/lib/travis/addons/archive.rb",
@ -696,16 +695,9 @@ Gem::Specification.new do |s|
"vendor/travis-core/lib/travis/api/v1/webhook/build/finished.rb",
"vendor/travis-core/lib/travis/api/v1/webhook/build/finished/job.rb",
"vendor/travis-core/lib/travis/api/v2.rb",
"vendor/travis-core/lib/travis/chunkifier.rb",
"vendor/travis-core/lib/travis/commit_command.rb",
"vendor/travis-core/lib/travis/config/database.rb",
"vendor/travis-core/lib/travis/config/defaults.rb",
"vendor/travis-core/lib/travis/config/url.rb",
"vendor/travis-core/lib/travis/engine.rb",
"vendor/travis-core/lib/travis/enqueue.rb",
"vendor/travis-core/lib/travis/enqueue/services.rb",
"vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb",
"vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb",
"vendor/travis-core/lib/travis/errors.rb",
"vendor/travis-core/lib/travis/event.rb",
"vendor/travis-core/lib/travis/event/config.rb",
@ -717,7 +709,6 @@ Gem::Specification.new do |s|
"vendor/travis-core/lib/travis/github.rb",
"vendor/travis-core/lib/travis/github/education.rb",
"vendor/travis-core/lib/travis/github/services.rb",
"vendor/travis-core/lib/travis/github/services/fetch_config.rb",
"vendor/travis-core/lib/travis/github/services/find_or_create_org.rb",
"vendor/travis-core/lib/travis/github/services/find_or_create_repo.rb",
"vendor/travis-core/lib/travis/github/services/find_or_create_user.rb",
@ -728,11 +719,6 @@ Gem::Specification.new do |s|
"vendor/travis-core/lib/travis/github/services/sync_user/repository.rb",
"vendor/travis-core/lib/travis/github/services/sync_user/reset_token.rb",
"vendor/travis-core/lib/travis/github/services/sync_user/user_info.rb",
"vendor/travis-core/lib/travis/logs.rb",
"vendor/travis-core/lib/travis/logs/services.rb",
"vendor/travis-core/lib/travis/logs/services/aggregate.rb",
"vendor/travis-core/lib/travis/logs/services/archive.rb",
"vendor/travis-core/lib/travis/logs/services/receive.rb",
"vendor/travis-core/lib/travis/mailer.rb",
"vendor/travis-core/lib/travis/mailer/user_mailer.rb",
"vendor/travis-core/lib/travis/mailer/views/layouts/contact_email.html.erb",
@ -798,13 +784,6 @@ Gem::Specification.new do |s|
"vendor/travis-core/lib/travis/notification/publisher/redis.rb",
"vendor/travis-core/lib/travis/overwritable_method_definitions.rb",
"vendor/travis-core/lib/travis/redis_pool.rb",
"vendor/travis-core/lib/travis/requests.rb",
"vendor/travis-core/lib/travis/requests/services.rb",
"vendor/travis-core/lib/travis/requests/services/receive.rb",
"vendor/travis-core/lib/travis/requests/services/receive/api.rb",
"vendor/travis-core/lib/travis/requests/services/receive/cron.rb",
"vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb",
"vendor/travis-core/lib/travis/requests/services/receive/push.rb",
"vendor/travis-core/lib/travis/secure_config.rb",
"vendor/travis-core/lib/travis/services.rb",
"vendor/travis-core/lib/travis/services/base.rb",
@ -859,7 +838,6 @@ Gem::Specification.new do |s|
"vendor/travis-core/lib/travis/testing/scenario.rb",
"vendor/travis-core/lib/travis/testing/stubs.rb",
"vendor/travis-core/lib/travis/testing/stubs/stub.rb",
"vendor/travis-core/lib/travis/travis_yml_stats.rb",
"vendor/travis-core/lib/travis_core/version.rb",
"vendor/travis-core/travis-core.gemspec"
>>>>>>> renamed the core spec directory

7
vendor/travis-core/lib/travis.rb vendored
View File

@ -43,14 +43,10 @@ module Travis
require 'travis/addons'
require 'travis/api'
require 'travis/config/defaults'
require 'travis/commit_command'
require 'travis/enqueue'
require 'travis/features'
require 'travis/github'
require 'travis/logs'
require 'travis/mailer'
require 'travis/notification'
require 'travis/requests'
require 'travis/services'
class UnknownRepository < StandardError; end
@ -71,10 +67,7 @@ module Travis
Github.setup
Addons.register
Services.register
Enqueue::Services.register
Github::Services.register
Logs::Services.register
Requests::Services.register
end
attr_accessor :redis, :config

16
vendor/travis-core/lib/travis/README.markdown vendored
View File

@ -1,16 +0,0 @@
# Travis Core directory overview
This folder, `lib/travis` contains the main code for the Travis Core repository. It contains several sub-section/subdirectories:
- [`addons`](addons): Event handlers that take events such as "build finished" and sends out notifications to GitHub, Pusher, Campfire, etc.
- [`api`](api): Serializers for models and events used in our API and in some other places (for example to generate Pusher payloads).
- [`enqueue`](enqueue): Logic for enqueueing jobs.
- [`event`](event): Code for sending and subscribing to events. Used by the `addons` code to subscribe to changes in the models.
- [`github`](github): Services for communicating with the GitHub API.
- [`mailer`](mailer): ActionMailer mailers.
- [`model`](model): All of our ActiveRecord models.
- [`notification`](notification): Code for adding instrumentation.
- [`requests`](requests): Handles requests received from GitHub.
- [`secure_config.rb`](secure_config.rb): Logic for encrypting and decrypting build/job configs.
- [`services`](services): Most of the business logic behind our [API](https://github.com/travis-ci/travis-api).
- [`testing`](testing): Code used by our tests, such as model stubs and factories.

59
vendor/travis-core/lib/travis/advisory_locks.rb vendored
View File

@ -1,59 +0,0 @@
require 'zlib'
module Travis
# http://hashrocket.com/blog/posts/advisory-locks-in-postgres
# https://github.com/mceachen/with_advisory_lock
# 13.3.4. Advisory Locks : http://www.postgresql.org/docs/9.3/static/explicit-locking.html
# http://www.postgresql.org/docs/9.3/static/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS
class AdvisoryLocks
attr_reader :lock_name, :transactional
def initialize(lock_name, transactional = false)
if lock_name.blank?
raise StandardError, "lock name cannot be blank"
end
@lock_name = lock_name
end
# must be used within a transaction
def self.exclusive(lock_name, timeout = 30, transactional = false)
al = self.new(lock_name, transactional)
al.exclusive(timeout) { yield }
end
# must be used within a transaction
def exclusive(timeout = 30)
give_up_at = Time.now + timeout if timeout
while timeout.nil? || Time.now < give_up_at do
if obtained_lock?
return yield
else
# Randomizing sleep time may help reduce contention.
sleep(rand(0.1..0.2))
end
end
ensure
release_lock unless transactional
end
private
def obtained_lock?
xact = transactional ? "_xact" : nil
result = connection.select_value("select pg_try_advisory#{xact}_lock(#{lock_code});")
result == 't' || result == 'true'
end
def release_lock
connection.execute("select pg_advisory_unlock(#{lock_code});")
end
def connection
ActiveRecord::Base.connection
end
def lock_code
Zlib.crc32(lock_name)
end
end
end

59
vendor/travis-core/lib/travis/chunkifier.rb vendored
View File

@ -1,59 +0,0 @@
require 'coder/cleaner/simple/encodings'
module Travis
class Chunkifier < Struct.new(:content, :chunk_size, :options)
include Enumerable
include Coder::Cleaner::Simple::Encodings::UTF_8
def initialize(*)
super
self.options ||= {}
end
def json?
options[:json]
end
def length
parts.length
end
def each(&block)
parts.each(&block)
end
def parts
@parts ||= split
end
def split
parts = content.scan(/.{1,#{chunk_split_size}}/m)
chunks = []
current_chunk = ''
parts.each do |part|
if too_big?(current_chunk + part)
chunks << current_chunk
current_chunk = part
else
current_chunk << part
end
end
chunks << current_chunk if current_chunk.length > 0
chunks
end
def chunk_split_size
size = chunk_size / 10
size == 0 ? 1 : size
end
def too_big?(current_chunk)
current_chunk = current_chunk.to_json if json?
current_chunk.bytesize > chunk_size
end
end
end

23
vendor/travis-core/lib/travis/commit_command.rb vendored
View File

@ -1,23 +0,0 @@
module Travis
class CommitCommand
def initialize(message)
@message = message.to_s
end
def skip?
backwards_skip or command == 'skip'
end
private
attr_reader :message
def command
message =~ /\[ci(?: |:)([\w ]*)\]/i && $1.downcase
end
def backwards_skip
message =~ /\[skip\s+ci\]/i && true
end
end
end

21
vendor/travis-core/lib/travis/engine.rb vendored
View File

@ -1,21 +0,0 @@
require 'travis'
require 'rails/engine'
module Travis
class Engine < Rails::Engine
initializer 'add migrations path' do |app|
# need to insert to both Rails.app.paths and Migrator.migration_paths
# because Rails' stupid rake tasks copy them over before loading the
# engines *unless* multiple rake db tasks are combined (as in rake
# db:create db:migrate). Happens in Rails <= 3.2.2
paths = [
Rails.application.paths['db/migrate'],
ActiveRecord::Migrator.migrations_paths
]
paths.each do |paths|
path = root.join('db/migrate').to_s
paths << path unless paths.include?(path)
end
end
end
end

6
vendor/travis-core/lib/travis/enqueue.rb vendored
View File

@ -1,6 +0,0 @@
module Travis
module Enqueue
require 'travis/enqueue/services'
end
end

14
vendor/travis-core/lib/travis/enqueue/services.rb vendored
View File

@ -1,14 +0,0 @@
module Travis
module Enqueue
module Services
require 'travis/enqueue/services/enqueue_jobs'
class << self
def register
constants(false).each { |name| const_get(name) }
end
end
end
end
end

127
vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb vendored
View File

@ -1,127 +0,0 @@
require 'travis/services/base'
require 'travis/support/instrumentation'
require 'travis/support/exceptions/handling'
module Travis
module Enqueue
module Services
# Finds owners that have queueable jobs and for each owner:
#
# * checks how many jobs can be enqueued
# * finds the oldest N queueable jobs and
# * enqueues them
class EnqueueJobs < Travis::Services::Base
TIMEOUT = 2
extend Travis::Instrumentation, Travis::Exceptions::Handling
require 'travis/enqueue/services/enqueue_jobs/limit'
register :enqueue_jobs
def self.run
new.run
end
def reports
@reports ||= {}
end
def run
enqueue_all && reports unless disabled?
end
instrument :run
rescues :run, from: Exception, backtrace: false
def disabled?
Timeout.timeout(TIMEOUT) do
Travis::Features.feature_deactivated?(:job_queueing)
end
rescue Timeout::Error, Redis::TimeoutError => e
Travis.logger.error("[enqueue] Timeout trying to check enqueuing feature flag.")
return false
end
private
def enqueue_all
grouped_jobs = jobs.group_by(&:owner)
Metriks.timer('enqueue.total').time do
grouped_jobs.each do |owner, jobs|
next unless owner
Metriks.timer('enqueue.full_enqueue_per_owner').time do
limit = nil
queueable = nil
Metriks.timer('enqueue.limit_per_owner').time do
Travis.logger.info "About to evaluate jobs for: #{owner.login}."
limit = Limit.new(owner, jobs)
queueable = limit.queueable
end
Metriks.timer('enqueue.enqueue_per_owner').time do
enqueue(queueable)
end
Metriks.timer('enqueue.report_per_owner').time do
reports[owner.login] = limit.report
end
end
end
end
end
def enqueue(jobs)
jobs.each do |job|
Travis.logger.info("enqueueing slug=#{job.repository.slug} job_id=#{job.id}")
Metriks.timer('enqueue.publish_job').time do
publish(job)
end
Metriks.timer('enqueue.enqueue_job').time do
job.enqueue
end
end
end
def publish(job)
Metriks.timer('enqueue.publish_job').time do
payload = Travis::Api.data(job, for: 'worker', type: 'Job::Test', version: 'v0')
publisher(job.queue).publish(payload, properties: { type: payload['type'], persistent: true })
end
end
def jobs
Metriks.timer('enqueue.fetch_jobs').time do
jobs = Job.includes(:owner).queueable.all
Travis.logger.info "Found #{jobs.size} jobs in total." if jobs.size > 0
jobs
end
end
def publisher(queue)
Travis::Amqp::Publisher.builds(queue)
end
class Instrument < Notification::Instrument
def run_completed
publish(msg: format(target.reports), reports: target.reports)
end
def format(reports)
reports = Array(reports)
if reports.any?
reports = reports.map do |repo, report|
" #{repo}: #{report.map { |key, value| "#{key}: #{value}" }.join(', ')}"
end
"enqueued:\n#{reports.join("\n")}"
else
'nothing to enqueue.'
end
end
end
Instrument.attach_to(self)
end
end
end
end

85
vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb vendored
View File

@ -1,85 +0,0 @@
require 'travis/services/base'
require 'travis/model/job'
module Travis
module Enqueue
module Services
class EnqueueJobs < Travis::Services::Base
class Limit
attr_reader :owner, :jobs, :config
def initialize(owner, jobs)
@owner = owner
@jobs = jobs
@config = Travis.config.queue.limit
end
def queueable
@queueable ||= filter_by_repository(jobs)[0, max_queueable]
end
def filter_by_repository(jobs)
return jobs unless Travis.config.limit_per_repo_enabled?
queueable_by_repository_id = {}
jobs.reject do |job|
if job.repository.settings.restricts_number_of_builds?
queueable?(job, queueable_by_repository_id, running_by_repository_id)
end
end
end
def running_by_repository_id
@running_by_repository ||= Hash[running_jobs.group_by(&:repository_id).map {|repository_id, jobs| [repository_id, jobs.size]}]
end
def queueable?(job, queueable, running)
repository = job.repository_id
queueable[repository] ||= 0
runnable_count = queueable[repository] +
(running[repository] || 0)
if runnable_count < job.repository.settings.maximum_number_of_builds
queueable[repository] += 1
false
else
true
end
end
def report
{ total: jobs.size, running: running, max: max_jobs, queueable: queueable.size }
end
private
def running_jobs
@running_jobs ||= Job.owned_by(owner).running
end
def running
@running ||= Job.owned_by(owner).running.count(:id)
end
def max_queueable
return config.default if owner.login.nil?
if unlimited?
999
else
queueable = max_jobs - running
queueable < 0 ? 0 : queueable
end
end
def max_jobs
config.by_owner[owner.login] || config.default
end
def unlimited?
config.by_owner[owner.login] == -1
end
end
end
end
end
end

1
vendor/travis-core/lib/travis/github/services.rb vendored
View File

@ -1,7 +1,6 @@
module Travis
module Github
module Services
require 'travis/github/services/fetch_config'
require 'travis/github/services/find_or_create_org'
require 'travis/github/services/find_or_create_repo'
require 'travis/github/services/find_or_create_user'

90
vendor/travis-core/lib/travis/github/services/fetch_config.rb vendored
View File

@ -1,90 +0,0 @@
require 'gh'
require 'yaml'
require 'active_support/core_ext/class/attribute'
require 'travis/support/logging'
require 'travis/support/instrumentation'
require 'travis/services/base'
module Travis
module Github
module Services
# encapsulates fetching a .travis.yml from a given commit's config_url
class FetchConfig < Travis::Services::Base
include Logging
extend Instrumentation
register :github_fetch_config
def run
config = retrying(3) { filter(parse(fetch)) }
config || Travis.logger.warn("[request:fetch_config] Empty config for request id=#{request.id} config_url=#{config_url.inspect}")
rescue GH::Error => e
if e.info[:response_status] == 404
{ '.result' => 'not_found' }
else
{ '.result' => 'server_error' }
end
end
instrument :run
def request
params[:request]
end
def config_url
request.config_url
end
private
def fetch
content = GH[config_url]['content']
Travis.logger.warn("[request:fetch_config] Empty content for #{config_url}") if content.nil?
content = content.to_s.unpack('m').first
Travis.logger.warn("[request:fetch_config] Empty unpacked content for #{config_url}, content was #{content.inspect}") if content.nil?
nbsp = "\xC2\xA0".force_encoding("binary")
content = content.gsub(/^(#{nbsp})+/) { |match| match.gsub(nbsp, " ") }
content
end
def parse(yaml)
YAML.load(yaml).merge('.result' => 'configured')
rescue StandardError, Psych::SyntaxError => e
error "[request:fetch_config] Error parsing .travis.yml for #{config_url}: #{e.message}"
{
'.result' => 'parse_error',
'.result_message' => e.is_a?(Psych::SyntaxError) ? e.message.split(": ").last : e.message
}
end
def filter(config)
unless Travis::Features.active?(:template_selection, request.repository)
config = config.to_h.except('dist').except('group')
end
config
end
def retrying(times)
count, result = 0, nil
until result || count > times
result = yield
count += 1
Travis.logger.warn("[request:fetch_config] Retrying to fetch config for #{config_url}") unless result
end
result
end
class Instrument < Notification::Instrument
def run_completed
# TODO exctract something like Url.strip_secrets
config_url = target.config_url.gsub(/(token|secret)=\w*/) { "#{$1}=[secure]" }
publish(msg: "#{config_url}", url: config_url, result: result)
end
end
Instrument.attach_to(self)
end
end
end
end

6
vendor/travis-core/lib/travis/logs.rb vendored
View File

@ -1,6 +0,0 @@
module Travis
module Logs
autoload :Services, 'travis/logs/services'
end
end

17
vendor/travis-core/lib/travis/logs/services.rb vendored
View File

@ -1,17 +0,0 @@
module Travis
module Logs
module Services
autoload :Aggregate, 'travis/logs/services/aggregate'
autoload :Archive, 'travis/logs/services/archive'
autoload :Receive, 'travis/logs/services/receive'
class << self
def register
constants(false).each { |name| const_get(name) }
end
end
end
end
end

96
vendor/travis-core/lib/travis/logs/services/aggregate.rb vendored
View File

@ -1,96 +0,0 @@
require 'active_support/core_ext/string/filters'
require 'travis/features'
require 'travis/model/log'
require 'travis/model/log/part'
require 'travis/services/base'
module Travis
module Logs
module Services
class Aggregate < Travis::Services::Base
register :logs_aggregate
AGGREGATE_UPDATE_SQL = <<-sql.squish
UPDATE logs
SET aggregated_at = ?,
content = (COALESCE(content, '') || (#{Log::AGGREGATE_PARTS_SELECT_SQL}))
WHERE logs.id = ?
sql
AGGREGATEABLE_SELECT_SQL = <<-sql.squish
SELECT DISTINCT log_id
FROM log_parts
WHERE created_at <= NOW() - interval '? seconds' AND final = ?
OR created_at <= NOW() - interval '? seconds'
sql
def run
return unless active?
aggregateable_ids.each do |id|
transaction do
aggregate(id)
vacuum(id)
notify(id)
end
end
end
private
def active?
Travis::Features.feature_active?(:log_aggregation)
end
def aggregate(id)
meter('logs.aggregate') do
connection.execute(sanitize_sql([AGGREGATE_UPDATE_SQL, Time.now, id, id]))
end
end
def vacuum(id)
meter('logs.vacuum') do
Log::Part.delete_all(log_id: id)
end
end
def notify(id)
Log.find(id).notify('aggregated')
rescue ActiveRecord::RecordNotFound
puts "[warn] could not find a log with the id #{id}"
end
def aggregateable_ids
Log::Part.connection.select_values(query).map { |id| id.nil? ? id : id.to_i }
end
def query
Log::Part.send(:sanitize_sql, [AGGREGATEABLE_SELECT_SQL, intervals[:regular], true, intervals[:force]])
end
def intervals
Travis.config.logs.intervals
end
def transaction(&block)
ActiveRecord::Base.transaction(&block)
rescue ActiveRecord::ActiveRecordError => e
# puts e.message, e.backtrace
Travis::Exceptions.handle(e)
end
def meter(name, &block)
Metriks.timer(name).time(&block)
end
def connection
Log::Part.connection
end
def sanitize_sql(*args)
Log::Part.send(:sanitize_sql, *args)
end
end
end
end
end

167
vendor/travis-core/lib/travis/logs/services/archive.rb vendored
View File

@ -1,167 +0,0 @@
begin
require 'aws/s3'
rescue LoadError => e
end
require 'uri'
require 'active_support/core_ext/hash/slice'
require 'faraday'
require 'travis/support/instrumentation'
require 'travis/notification/instrument'
require 'travis/services/base'
module Travis
class S3
class << self
def setup
AWS.config(Travis.config.s3.to_h.slice(:access_key_id, :secret_access_key))
end
end
attr_reader :s3, :url
def initialize(url)
@s3 = AWS::S3.new
@url = url
end
def store(data)
object.write(data, content_type: 'text/plain', acl: :public_read)
end
def object
@object ||= bucket.objects[URI.parse(url).path[1..-1]]
end
def bucket
@bucket ||= s3.buckets[URI.parse(url).host]
end
end
module Logs
module Services
class Archive < Travis::Services::Base
class FetchFailed < StandardError
def initialize(source_url, status, message)
super("Could not retrieve #{source_url}. Response status: #{status}, message: #{message}")
end
end
class VerificationFailed < StandardError
def initialize(source_url, target_url, expected, actual)
super("Expected #{target_url} (from: #{source_url}) to have the content length #{expected.inspect}, but had #{actual.inspect}")
end
end
extend Travis::Instrumentation
register :archive_log
attr_reader :log
def run
fetch
store
verify
report
end
instrument :run
def source_url
"https://#{hostname('api')}/logs/#{params[:id]}.txt"
end
def report_url
"https://#{hostname('api')}/logs/#{params[:id]}"
end
def target_url
"http://#{hostname('archive')}/jobs/#{params[:job_id]}/log.txt"
end
private
def fetch
retrying(:fetch) do
response = request(:get, source_url)
if response.status == 200
@log = response.body.to_s
else
raise(FetchFailed.new(source_url, response.status, response.body.to_s))
end
end
end
def store
retrying(:store) do
S3.setup
s3.store(log)
end
end
def verify
retrying(:verify) do
expected = log.bytesize
actual = request(:head, target_url).headers['content-length'].try(:to_i)
raise VerificationFailed.new(target_url, source_url, expected, actual) unless expected == actual
end
end
def report
retrying(:report) do
request(:put, report_url, { archived_at: Time.now, archive_verified: true }, token: Travis.config.tokens.internal)
end
end
def request(method, url, params = nil, headers = nil, &block)
http.send(*[method, url, params, headers].compact, &block)
rescue Faraday::Error => e
puts "Exception while trying to #{method.inspect}: #{source_url}:"
puts e.message, e.backtrace
raise e
end
def http
Faraday.new(ssl: Travis.config.ssl.to_h.compact) do |f|
f.request :url_encoded
f.adapter :net_http
end
end
def s3
S3.new(target_url)
end
def hostname(name)
"#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}"
end
def retrying(header, times = 5)
yield
rescue => e
count ||= 0
if times > (count += 1)
puts "[#{header}] retry #{count} because: #{e.message}"
Travis::Instrumentation.meter("#{self.class.name.underscore.gsub("/", ".")}.retries.#{header}")
sleep count * 3 unless params[:no_sleep]
retry
else
raise
end
end
class Instrument < Notification::Instrument
def run_completed
publish(
msg: "for <Log id=#{target.params[:id]}> (to: #{target.target_url})",
source_url: target.source_url,
target_url: target.target_url,
object_type: 'Log',
object_id: target.params[:id]
)
end
end
Instrument.attach_to(self)
end
end
end
end

79
vendor/travis-core/lib/travis/logs/services/receive.rb vendored
View File

@ -1,79 +0,0 @@
require 'coder'
require 'metriks'
require 'travis/model/log/part'
require 'travis/model/job/test'
require 'travis/services/base'
module Travis
module Logs
module Services
class Receive < Travis::Services::Base
# TODO remove this once we know aggregation works fine and the worker passes a :final flag
FINAL = 'Done. Build script exited with:'
register :logs_receive
def run
create_part
notify
end
private
def create_part
measure('logs.update') do
Travis.logger.warn "[warn] log.id is #{log.id.inspect} in :logs_append! job_id: #{data[:id]}" if log.id.to_i == 0
Log::Part.create!(log_id: log.id, content: chars, number: number, final: final?)
end
rescue ActiveRecord::ActiveRecordError => e
Travis.logger.warn "[warn] could not save log in :logs_append job_id: #{data[:id]}: #{e.message}"
Travis.logger.warn e.backtrace
end
def notify
job.notify(:log, _log: chars, number: number, final: final?)
rescue => e
Metriks.meter('travis.logs.update.notify.errors').mark
Travis.logger.error("Error notifying of log update: #{e.message} (from #{e.backtrace.first})")
end
def log
@log ||= Log.where(job_id: job.id).select(:id).first || create_log
end
def create_log
Travis.logger.warn "[warn] Had to create a log for job_id: #{job.id}!"
job.create_log!
end
def job
@job ||= Job::Test.find(data[:id])
end
def chars
@chars ||= filter(data[:log])
end
def number
data[:number]
end
def final?
!!data[:final] || chars.include?(FINAL)
end
def data
@data ||= params[:data].symbolize_keys
end
def filter(chars)
Coder.clean!(chars.to_s.gsub("\0", '')) # postgres seems to have issues with null chars
end
def measure(name, &block)
Metriks.timer(name).time(&block)
end
end
end
end
end

6
vendor/travis-core/lib/travis/requests.rb vendored
View File

@ -1,6 +0,0 @@
module Travis
module Requests
require 'travis/requests/services'
end
end

13
vendor/travis-core/lib/travis/requests/services.rb vendored
View File

@ -1,13 +0,0 @@
module Travis
module Requests
module Services
require 'travis/requests/services/receive'
class << self
def register
constants(false).each { |name| const_get(name) }
end
end
end
end
end

207
vendor/travis-core/lib/travis/requests/services/receive.rb vendored
View File

@ -1,207 +0,0 @@
require 'gh'
require 'travis/services/base'
require 'travis/model/request/approval'
require 'travis/notification/instrument'
require 'travis/advisory_locks'
require 'travis/travis_yml_stats'
module Travis
module Requests
module Services
class Receive < Travis::Services::Base
require 'travis/requests/services/receive/api'
require 'travis/requests/services/receive/cron'
require 'travis/requests/services/receive/pull_request'
require 'travis/requests/services/receive/push'
extend Travis::Instrumentation
class PayloadValidationError < StandardError; end
register :receive_request
class << self
def payload_for(type, data)
data = GH.load(data)
const_get(type.camelize).new(data)
end
end
attr_reader :request, :accepted
def run
with_transactional_advisory_lock do
if accept?
create && start
store_config_info if verify
else
rejected
end
request
end
rescue GH::Error => e
Travis.logger.error "payload for #{slug} could not be received as GitHub returned a #{e.info[:response_status]}: #{e.info}, github-guid=#{github_guid}, event-type=#{event_type}"
end
instrument :run
def accept?
payload.validate!
validate!
@accepted = payload.accept?
rescue PayloadValidationError => e
Travis.logger.error "#{e.message}, github-guid=#{github_guid}, event-type=#{event_type}"
@accepted = false
end
private
def with_transactional_advisory_lock
return yield unless payload.repository
result = nil
Travis::AdvisoryLocks.exclusive("receive-repo:#{payload.repository[:github_id]}", 300) do
ActiveRecord::Base.connection.transaction do
result = yield
end
end
result
rescue => e
ActiveRecord::Base.connection.rollback_db_transaction
raise
end
def validate!
repo_not_found! unless repo
verify_owner
end
def verify_owner
owner = owner_by_payload
owner_not_found! unless owner
update_owner(owner) if owner.id != repo.owner_id && !api_request?
end
def create
@request = repo.requests.create!(payload.request.merge(
:payload => params[:payload],
:event_type => event_type,
:state => :created,
:commit => commit,
:owner => repo.owner,
:token => params[:token]
))
end
def start
request.start!
end
def verify
request.reload
if request.builds.count == 0
approval = Request::Approval.new(request)
Travis.logger.warn("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} didn't create any builds: #{approval.result}/#{approval.message}")
false
elsif !request.creates_jobs?
approval = Request::Approval.new(request)
Travis.logger.warn("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} didn't create any job: #{approval.result}/#{approval.message}")
false
else
Travis.logger.info("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} created #{request.builds.count} builds")
true
end
end
def update_owner(owner)
repo.update_attributes!(owner: owner, owner_name: owner.login)
owner_updated
end
def owner_by_payload
if id = payload.repository[:owner_id]
lookup_owner(payload.repository[:owner_type], id: id)
elsif github_id = payload.repository[:owner_github_id]
lookup_owner(payload.repository[:owner_type], github_id: github_id)
elsif login = payload.repository[:owner_name]
lookup_owner(%w(User Organization), login: login)
end
end
def lookup_owner(types, attrs)
Array(types).map(&:constantize).each do |type|
owner = type.where(attrs).first
return owner if owner
end
nil
end
def repo_not_found!
Travis::Metrics.meter('request.receive.repository_not_found')
raise PayloadValidationError, "Repository not found: #{payload.repository.slice(:id, :github_id, :owner_name, :name)}"
end
def owner_not_found!
Travis::Metrics.meter('request.receive.repository_owner_not_found')
raise PayloadValidationError, "The given repository owner could not be found: #{payload.repository.slice(:owner_id, :owner_github_id, :owner_type, :owner_name).inspect}"
end
def owner_updated
Travis::Metrics.meter('request.receive.update_owner')
Travis.logger.warn("[request:receive] Repository owner updated for #{slug}: #{repo.owner_type}##{repo.owner_id} (#{repo.owner_name})")
end
def rejected
commit = payload.commit['commit'].inspect if payload.commit rescue nil
Travis.logger.info("[request:receive] Github event rejected: event_type=#{event_type.inspect} repo=\"#{slug}\" commit=#{commit} action=#{payload.action.inspect}")
end
def payload
@payload ||= self.class.payload_for(event_type, params[:payload])
end
def github_guid
params[:github_guid]
end
def event_type
@event_type ||= (params[:event_type] || 'push').gsub('-', '_')
end
def api_request?
event_type == 'api'
end
def repo
@repo ||= run_service(:find_repo, payload.repository)
end
def slug
payload.repository ? payload.repository.values_at(:owner_name, :name).join('/') : '?'
end
def commit
@commit ||= repo.commits.create!(payload.commit) if payload.commit
end
def store_config_info
Travis::TravisYmlStats.store_stats(request)
rescue => e
Travis.logger.warn("[request:receive] Couldn't store .travis.yml stats: #{e.message}")
Travis::Exceptions.handle(e)
end
class Instrument < Notification::Instrument
def run_completed
params = target.params
publish(
:msg => "type=#{params[:event_type].inspect}",
:type => params[:event_type],
:accept? => target.accepted,
:payload => params[:payload]
)
end
end
Instrument.attach_to(self)
end
end
end
end

106
vendor/travis-core/lib/travis/requests/services/receive/api.rb vendored
View File

@ -1,106 +0,0 @@
module Travis
module Requests
module Services
class Receive < Travis::Services::Base
class Api
VALIDATION_ERRORS = {
repo: 'Repository data is not present in payload',
}
attr_reader :event
def initialize(event)
@event = event
end
def accept?
true
end
def validate!
error(:repo) if repo_data.nil?
end
def action
nil
end
def repository
@repository ||= {
owner_id: repo_data['owner_id'],
owner_type: repo_data['owner_type'],
owner_name: repo_data['owner_name'],
name: repo_data['name']
}
end
def request
@request ||= {
:config => event['config']
}
end
def commit
@commit ||= {
commit: commit_data['sha'],
message: message,
branch: branch,
ref: nil, # TODO verify that we do not need this
committed_at: commit_data['commit']['committer']['date'], # TODO in case of API requests we'd want to display the timestamp of the incoming request
committer_name: commit_data['commit']['committer']['name'],
committer_email: commit_data['commit']['committer']['email'],
author_name: commit_data['commit']['author']['name'],
author_email: commit_data['commit']['author']['email'],
compare_url: commit_data['_links']['self']['href']
}
end
private
def gh
Github.authenticated(user)
end
def user
@user ||= User.find(event['user']['id'])
end
def repo_data
event['repository'] || {}
end
def message
event['message'] || commit_data['commit']['message']
end
def slug
repo_data.values_at('owner_name', 'name').join('/')
end
def branch
event['branch'] || 'master'
end
def repo_github_id
repo.try(:github_id) || raise(ActiveRecord::RecordNotFound)
end
def repo
if id = repo_data['id']
Repository.find(id)
else
Repository.by_slug(slug).first
end
end
def commit_data
@commit_data ||= gh["repos/#{slug}/commits?sha=#{branch}&per_page=1"].first # TODO I guess Api would protect against GH errors?
end
def error(type)
raise PayloadValidationError, VALIDATION_ERRORS[type]
end
end
end
end
end
end

10
vendor/travis-core/lib/travis/requests/services/receive/cron.rb vendored
View File

@ -1,10 +0,0 @@
module Travis
module Requests
module Services
class Receive < Travis::Services::Base
class Cron < Api
end
end
end
end
end

117
vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb vendored
View File

@ -1,117 +0,0 @@
module Travis
module Requests
module Services
class Receive < Travis::Services::Base
class PullRequest
attr_reader :event
def initialize(event)
@event = event
end
def accept?
return false if disabled? || closed?
case action
when :opened, :reopened then !!merge_commit
when :synchronize then head_change?
else false
end
end
def validate!
if event['repository'].nil?
raise PayloadValidationError, "Repository data is not present in payload"
end
end
def disabled?
Travis::Features.feature_deactivated?(:pull_requests)
end
def closed?
pull_request['state'] == 'closed'
end
def head_change?
head_commit && ::Request.last_by_head_commit(head_commit['sha']).nil?
end
def repository
@repository ||= repo && {
name: repo['name'],
description: repo['description'],
url: repo['_links']['html']['href'],
owner_github_id: repo['owner']['id'],
owner_type: repo['owner']['type'],
owner_name: repo['owner']['login'],
owner_email: repo['owner']['email'],
private: !!repo['private'],
github_id: repo['id']
}
end
def request
@request ||= {
comments_url: comments_url,
base_commit: base_commit['sha'],
head_commit: head_commit['sha']
}
end
def commit
@commit ||= if merge_commit
{
commit: merge_commit['sha'],
message: head_commit['message'],
branch: pull_request['base']['ref'],
ref: merge_commit['ref'],
committed_at: committer['date'],
committer_name: committer['name'],
committer_email: committer['email'],
author_name: author['name'],
author_email: author['email'],
compare_url: pull_request['_links']['html']['href']
}
end
end
def pull_request
event['pull_request'] || {}
end
def action
event['action'].try(:to_sym)
end
def comments_url
pull_request.fetch('_links', {}).fetch('comments', {}).fetch('href', '')
end
def base_commit
pull_request['base_commit'] || { 'sha' => '' }
end
def head_commit
pull_request['head_commit']
end
def merge_commit
pull_request['merge_commit']
end
def repo
event['repository']
end
def committer
head_commit.fetch('committer', {})
end
def author
head_commit.fetch('author', {})
end
end
end
end
end
end

84
vendor/travis-core/lib/travis/requests/services/receive/push.rb vendored
View File

@ -1,84 +0,0 @@
module Travis
module Requests
module Services
class Receive < Travis::Services::Base
class Push
attr_reader :event
def initialize(event)
@event = event
end
def accept?
true
end
def validate!
if event['repository'].nil?
raise PayloadValidationError, "Repository data is not present in payload"
end
end
def action
nil
end
def repository
@repository ||= repo_data && {
name: repo_data['name'],
description: repo_data['description'],
url: repo_data['_links']['html']['href'],
owner_github_id: repo_data['owner']['id'],
owner_type: repo_data['owner']['type'],
owner_name: repo_data['owner']['login'],
owner_email: repo_data['owner']['email'],
private: !!repo_data['private'],
github_id: repo_data['id']
}
end
def request
@request ||= {}
end
def commit
@commit ||= commit_data && {
commit: commit_data['sha'],
message: commit_data['message'],
branch: event['ref'].split('/', 3).last,
ref: event['ref'],
committed_at: commit_data['date'],
committer_name: commit_data['committer']['name'],
committer_email: commit_data['committer']['email'],
author_name: commit_data['author']['name'],
author_email: commit_data['author']['email'],
compare_url: event['compare']
}
end
private
def repo_data
event['repository']
end
def commit_data
last_unskipped_commit || commits.last || event['head_commit']
end
def last_unskipped_commit
commits.reverse.find { |commit| !skip_commit?(commit) }
end
def commits
event['commits'] || []
end
def skip_commit?(commit)
Travis::CommitCommand.new(commit['message']).skip?
end
end
end
end
end
end

195
vendor/travis-core/lib/travis/travis_yml_stats.rb vendored
View File

@ -1,195 +0,0 @@
require "sidekiq"
begin
require "keen"
rescue LoadError
end
module Travis
class TravisYmlStats
class KeenPublisher
include ::Sidekiq::Worker
sidekiq_options queue: :keen_events
def perform(payload, deployment_payload = nil, notification_payload = nil)
if defined?(Keen) && ENV["KEEN_PROJECT_ID"]
payload = { :requests => [payload] }
payload[:deployments] = deployment_payload if deployment_payload.to_a.size > 0
payload[:notifications] = notification_payload if notification_payload.to_a.size > 0
Keen.publish_batch(payload)
end
end
end
LANGUAGE_VERSION_KEYS = %w[
ghc
go
jdk
node_js
otp_release
perl
php
python
ruby
rvm
scala
]
def self.store_stats(request, publisher=KeenPublisher)
new(request, publisher).store_stats
end
def initialize(request, publisher)
@request = request
@publisher = publisher
@keen_payload = {}
@keen_payload_deployment = []
@keen_payload_notification = []
end
def store_stats
set_basic_info
set_language
set_language_version
set_uses_sudo
set_uses_apt_get
set_dist
set_group
set_deployment_provider_count
set_notification
@publisher.perform_async(keen_payload, keen_payload_deployment, keen_payload_notification)
end
private
attr_reader :request, :keen_payload
attr_accessor :keen_payload_deployment
attr_accessor :keen_payload_notification
def set(path, value, collection = keen_payload)
path = Array(path)
hsh = collection
path[0..-2].each do |key|
hsh[key.to_sym] ||= {}
hsh = hsh[key.to_sym]
end
hsh[path.last.to_sym] = value
end
def set_basic_info
set :event_type, request.event_type
set :matrix_size, request.builds.map { |build| build.matrix.size }.reduce(:+)
set :repository_id, request.repository_id
set :owner_id, request.owner_id
set :owner_type, request.owner_type
# The owner_type, owner_id tuple is there so we can do unique counts on it
set :owner, [request.owner_type, request.owner_id]
end
def set_language
set :language, travis_yml_language
set :github_language, github_language
end
def set_language_version
LANGUAGE_VERSION_KEYS.each do |key|
if config.key?(key)
case config[key]
when String, Array
set [:language_version, key], Array(config[key]).map(&:to_s).sort
else
set [:language_version, key], ["invalid"]
end
end
end
end
def set_uses_sudo
set :uses_sudo, commands.any? { |command| command =~ /\bsudo\b/ }
end
def set_uses_apt_get
set :uses_apt_get, commands.any? { |command| command =~ /\bapt-get\b/ }
end
def set_dist
set :dist_name, dist_name
end
def set_group
set :group_name, group_name
end
def set_deployment_provider_count
deploy = config["deploy"] || return
# Hash#to_a is not what we want here
deployments = deploy.is_a?(Hash) ? [deploy] : Array(deploy)
deployments.map {|d| d["provider"] }.uniq.each do |provider|
keen_payload_deployment << { provider: provider.downcase, repository_id: request.repository_id }
end
rescue
nil
end
def set_notification
notifications = config["notifications"] || return
notifications.keys.each do |notifier|
keen_payload_notification << { notifier: notifier.downcase, repository_id: request.repository_id }
end
rescue
nil
end
def config
request.config
end
def payload
request.payload.is_a?(String) ? MultiJson.decode(request.payload) : request.payload
end
def commands
[
config["before_install"],
config["install"],
config["before_script"],
config["script"],
config["after_success"],
config["after_failure"],
config["before_deploy"],
config["after_deploy"],
].flatten.compact
end
def travis_yml_language
language = config["language"]
case language
when String
language
when nil
"default"
else
"invalid"
end
end
def github_language
payload.fetch("repository", {})["language"]
end
def normalize_string(str)
str.downcase.gsub("#", "-sharp").gsub(/[^A-Za-z0-9.:\-_]/, "")
end
def dist_name
config.fetch('dist', 'default')
end
def group_name
config.fetch('group', 'default')
end
end
end