delete github/services/sync_user and user_mailer, remove action_mailer

This commit is contained in:
Sven Fuchs 2016-06-19 15:04:39 +02:00
parent a6658fa4d3
commit 0f1e697abd
17 changed files with 10 additions and 902 deletions

View File

@ -69,7 +69,6 @@ PATH
remote: .
specs:
travis-api (0.0.1)
actionmailer (~> 3.2.19)
activerecord (~> 3.2.19)
coder (~> 0.4.0)
composite_primary_keys (~> 5.0)

View File

@ -1,8 +1,9 @@
require 'travis/api/app'
require 'addressable/uri'
require 'faraday'
require 'securerandom'
require 'customerio'
require 'travis/api/app'
require 'travis/github/education'
class Travis::Api::App
class Endpoint

View File

@ -38,6 +38,10 @@ RSpec.configure do |c|
DatabaseCleaner.clean_with :truncation
DatabaseCleaner.strategy = :transaction
# This sets up a scenario in the db as an initial state. The db will be
# rolled back to this state after each test. Several tests in ./spec depend
# on this scenario, so this gives a performance benefit, but also can be
# confusing.
Scenario.default
end
@ -45,7 +49,6 @@ RSpec.configure do |c|
DatabaseCleaner.start
Redis.new.flushall
Travis.config.oauth2.scope = "user:email,public_repo"
# set_app Travis::Api::App.new
end
c.before :each, set_app: true do
@ -53,7 +56,6 @@ RSpec.configure do |c|
end
c.after :each do
# puts DatabaseCleaner.connections.map(&:strategy).map(&:class).map(&:name).join(', ')
DatabaseCleaner.clean
custom_endpoints.each do |endpoint|
endpoint.superclass.direct_subclasses.delete(endpoint)

View File

@ -25,7 +25,6 @@ Gem::Specification.new do |s|
# from travis-core gemspec
s.add_dependency 'activerecord', '~> 3.2.19'
s.add_dependency 'actionmailer', '~> 3.2.19'
s.add_dependency 'railties', '~> 3.2.19'
s.add_dependency 'rollout', '~> 1.1.0'
s.add_dependency 'coder', '~> 0.4.0'

View File

@ -1,7 +1,7 @@
require 'pusher'
require 'travis/support'
require 'travis/support/database'
require 'travis_core/version'
require 'travis/version'
require 'travis/redis_pool'
require 'travis/errors'
require 'travis/commit_command'
@ -46,7 +46,6 @@ module Travis
require 'travis/config/defaults'
require 'travis/features'
require 'travis/github'
require 'travis/mailer'
require 'travis/notification'
require 'travis/services'

View File

@ -1,16 +1,16 @@
require 'gh'
require 'core_ext/hash/compact'
require 'travis/github/education'
require 'travis/github/services'
module Travis
module Github
require 'travis/github/services'
class << self
def setup
GH.set(
client_id: Travis.config.oauth2.client_id,
client_secret: Travis.config.oauth2.client_secret,
user_agent: "Travis-CI/#{TravisCore::VERSION} GH/#{GH::VERSION}",
user_agent: "Travis-CI/#{Travis::VERSION} GH/#{GH::VERSION}",
origin: Travis.config.host,
api_url: Travis.config.github.api_url,
ssl: Travis.config.ssl.to_h.merge(Travis.config.github.ssl || {}).to_h.compact

View File

@ -5,7 +5,6 @@ module Travis
require 'travis/github/services/find_or_create_repo'
require 'travis/github/services/find_or_create_user'
require 'travis/github/services/set_hook'
require 'travis/github/services/sync_user'
class << self
def register

View File

@ -1,74 +0,0 @@
require 'metriks'
require 'travis/mailer/user_mailer'
require 'travis/services/base'
module Travis
module Github
module Services
class SyncUser < Travis::Services::Base
require 'travis/github/services/sync_user/organizations'
require 'travis/github/services/sync_user/repositories'
require 'travis/github/services/sync_user/repository'
require 'travis/github/services/sync_user/reset_token'
require 'travis/github/services/sync_user/user_info'
register :github_sync_user
def run
new_user? do
syncing do
# if Time.now.utc.tuesday? && Travis::Features.feature_active?("reset_token_in_sync")
# ResetToken.new(user).run
# end
UserInfo.new(user).run
Organizations.new(user).run
Repositories.new(user).run
end
end
ensure
user.update_column(:is_syncing, false)
end
def user
# TODO check that clients are only passing the id
@user ||= current_user || User.find(params[:id])
end
def new_user?
new_user = user.synced_at.nil? && user.created_at > 48.hours.ago.utc
yield if block_given?
if new_user and Travis.config.welcome_email
send_welcome_email
end
end
def send_welcome_email
return unless user.email.present?
UserMailer.welcome_email(user).deliver
logger.info("Sent welcome email to #{user.login}")
Metriks.meter('travis.welcome.email').mark
end
private
def syncing
unless user.github_oauth_token?
logger.warn "user sync for #{user.login} (id:#{user.id}) was cancelled as the user doesn't have a token"
return
end
user.update_column(:is_syncing, true)
result = yield
user.update_column(:synced_at, Time.now)
result
rescue GH::TokenInvalid => e
logger.warn "user sync for #{user.login} (id:#{user.id}) failed as the token was invalid, dropping the token"
user.update_column(:github_oauth_token, nil)
ensure
user.update_column(:is_syncing, false)
end
end
end
end
end

View File

@ -1,143 +0,0 @@
require 'gh'
module Travis
module Github
module Services
class SyncUser < Travis::Services::Base
class Organizations
class Filter
attr_reader :data, :limit
def initialize(data, options = {})
@data = data || {}
@limit = options[:repositories_limit] || 1000
end
def allow?
repositories_count < limit
end
def repositories_count
# I was not sure how to handle the case where we don't get the
# sufficient amount of data here and this seems the best answer,
# that way we will not get orgs siltently ignored
data['public_repositories'] || 0
end
end
class << self
def cancel_memberships(user, orgs)
user.memberships.where(:organization_id => orgs.map(&:id)).delete_all
end
end
extend Travis::Instrumentation
include Travis::Logging
attr_reader :user, :data
def initialize(user)
@user = user
end
def run
with_github do
{ :synced => create_or_update, :removed => remove }
end
end
instrument :run
private
def create_or_update
fetch_and_filter.map do |data|
org = create_or_update_org(data)
user.organizations << org unless user.organizations.include?(org)
org
end
end
def remove
orgs = user.organizations.reject { |org| github_ids.include?(org.github_id) }
self.class.cancel_memberships(user, orgs)
orgs
end
def fetch
@data ||= GH['user/orgs'].to_a
end
instrument :fetch, :level => :debug
def github_ids
@github_ids ||= data.map { |org| org['id'] }
end
def with_github(&block)
# TODO in_parallel should return the block's result in a future version
result = nil
GH.with(:token => user.github_oauth_token) do
# GH.in_parallel do
result = yield
# end
end
result
end
def fetch_and_filter
fetch.map do |data|
fetch_resource("organizations/#{data['id']}")
end.find_all do |data|
options = Travis.config.sync.organizations || {}
Filter.new(data, options).allow?
end
end
def fetch_resource(resource)
GH[resource] # TODO should be: ?type=#{self.class.type} but GitHub doesn't work as documented
rescue GH::Error => e
log_exception(e)
end
def create_or_update_org(data)
org = Organization.find_or_create_by_github_id(data['id'])
org.update_attributes!({
:name => data['name'],
:login => data['login'],
:email => data['email'],
:avatar_url => avatar_url(data['_links']['avatar']),
:location => data['location'],
:homepage => data['_links']['blog'].try(:fetch, 'href'),
:company => data['company']
})
org
end
def avatar_url(github_data)
href = github_data.try(:fetch, 'href')
href ? href[/^(https:\/\/[\w\.\/]*)/, 1] : nil
end
class Instrument < Notification::Instrument
def run_completed
format = lambda do |orgs|
orgs.map { |org| { id: org.id, login: org.login } }
end
publish(
msg: %(for #<User id=#{target.user.id} login="#{target.user.login}">),
result: { synced: format.call(result[:synced]), removed: format.call(result[:removed]) }
)
end
def fetch_completed
publish(
msg: %(for #<User id=#{target.user.id} login="#{target.user.login}">),
result: result
)
end
end
Instrument.attach_to(self)
end
end
end
end
end

View File

@ -1,140 +0,0 @@
require 'active_support/core_ext/class/attribute'
module Travis
module Github
module Services
class SyncUser < Travis::Services::Base
# Fetches all repositories from Github which are in /user/repos or any of the user's
# orgs/[name]/repos. Creates or updates existing repositories on our side and adds
# it to the user's permissions. Also removes existing permissions for repositories
# which are not in the received Github data. NOTE that this does *not* delete any
# repositories because we do not know if the repository was deleted or renamed
# on Github's side.
class Repositories
extend Travis::Instrumentation
include Travis::Logging
class_attribute :types
self.types = [:public]
class << self
# TODO backwards compat, remove once all apps use `types=`
def type=(types)
self.types = Array.wrap(types).map(&:to_s).join(',').split(',').map(&:to_sym)
end
def include?(type)
self.types.include?(type)
end
end
attr_reader :user, :resources, :data
def initialize(user)
@user = user
@resources = ['user/repos'] + user.organizations.map { |org| "orgs/#{org.login}/repos" }
end
def run
with_github do
{ :synced => create_or_update, :removed => remove }
end
end
instrument :run
private
def create_or_update
data.map do |repository|
Repository.new(user, repository).run
end
end
def remove
repos = user.repositories.reject { |repo| slugs.include?(repo.slug) }
Repository.unpermit_all(user, repos)
repos
end
# we have to filter these ourselves because the github api is broken for this
def data
@data ||= filter_duplicates(filter_based_on_repo_permission)
end
def filter_based_on_repo_permission
fetch.select { |repo| self.class.include?(repo['private'] ? :private : :public) }
end
def filter_duplicates(repositories)
repositories.each_with_object([]) do |repository, filtered_list|
unless in_filtered_list?(filtered_list, repository)
filtered_list.push(repository)
end
end
end
def in_filtered_list?(filtered_list, other_repository)
filtered_list.any? do |existing_repository|
same_repository_with_admin?(existing_repository, other_repository)
end
end
def same_repository_with_admin?(existing_repository, other_repository)
existing_repository['owner']['login'] == other_repository['owner']['login'] and
existing_repository['name'] == other_repository['name'] and
existing_repository['permissions']['admin'] == true
end
def slugs
@slugs ||= data.map { |repo| "#{repo['owner']['login']}/#{repo['name']}" }
end
def fetch
resources.map { |resource| fetch_resource(resource) }.map(&:to_a).flatten.compact
end
instrument :fetch, :level => :debug
def fetch_resource(resource)
GH[resource] # TODO should be: ?type=#{self.class.type} but GitHub doesn't work as documented
rescue GH::Error => e
log_exception(e)
end
def with_github(&block)
# TODO in_parallel should return the block's result in a future version
result = nil
GH.with(:token => user.github_oauth_token) do
# GH.in_parallel do
result = yield
# end
end
result
end
class Instrument < Notification::Instrument
def run_completed
format = lambda do |repos|
repos.map { |repo| { id: repo.id, owner: repo.owner_name, name: repo.name } }
end
publish(
msg: %(for #<User id=#{target.user.id} login="#{target.user.login}">),
resources: target.resources,
result: { synced: format.call(result[:synced]), removed: format.call(result[:removed]) }
)
end
def fetch_completed
publish(
msg: %(for #<User id=#{target.user.id} login="#{target.user.login}">),
resources: target.resources,
result: result
)
end
end
Instrument.attach_to(self)
end
end
end
end
end

View File

@ -1,140 +0,0 @@
module Travis
module Github
module Services
class SyncUser < Travis::Services::Base
class Repository
class << self
def unpermit_all(user, repositories)
user.permissions.where(:repository_id => repositories.map(&:id)).delete_all unless repositories.empty?
end
end
attr_reader :user, :data, :repo
def initialize(user, data)
@user = user
@data = data
end
def run
@repo = find || create
update
if permission
sync_permissions
elsif permit?
permit
end
repo
end
private
def find
::Repository.where(:github_id => github_id).first
end
def create
if Travis::Features.enabled_for_all?(:sync_repo_owner)
::Repository.create!(:owner => owner, :owner_name => owner_name, :name => name, github_id: github_id)
else
::Repository.create!(:owner_name => owner_name, :name => name, github_id: github_id)
end
end
# instrument :create, :level => :debug
def permission
@permission ||= user.permissions.where(:repository_id => repo.id).first
end
def sync_permissions
if permit?
permission.update_attributes!(permission_data)
else
permission.destroy
end
end
def permit?
push_access? || admin_access? || repo.private?
end
def permit
user.permissions.create!({
:user => user,
:repository => repo
}.merge(permission_data))
end
# instrument :permit, :level => :debug
def update
if Travis::Features.enabled_for_all?(:sync_repo_owner)
repo.update_attributes!({
owner: owner,
github_id: data['id'],
private: data['private'],
description: data['description'],
url: data['homepage'],
default_branch: data['default_branch'],
github_language: data['language'],
name: name,
owner_name: owner_name
})
else
repo.update_attributes!({
github_id: data['id'],
private: data['private'],
description: data['description'],
url: data['homepage'],
default_branch: data['default_branch'],
github_language: data['language'],
name: name,
owner_name: owner_name
})
end
rescue ActiveRecord::RecordInvalid
# ignore for now. this seems to happen when multiple syncs (i.e. user sign
# in requests are running in parallel?
rescue GH::Error(response_status: 404) => e
Travis.logger.warn "[github][services][user_sync] GitHub info was not available for #{repo.owner_name}/#{repo.name}: #{e.inspect}"
end
def owner
@owner ||= owner_type.constantize.find_by_github_id(owner_id)
end
def owner_id
data['owner']['id']
end
def owner_type
data['owner']['type']
end
def owner_name
data['owner']['login']
end
def name
data['name']
end
def github_id
data['id']
end
def permission_data
data['permissions']
end
def push_access?
permission_data['push']
end
def admin_access?
permission_data['admin']
end
end
end
end
end
end

View File

@ -1,36 +0,0 @@
require "gh"
module Travis
module Github
module Services
class SyncUser < Travis::Services::Base
class ResetToken
def initialize(user, config = Travis.config.oauth2.to_h, gh = nil)
@user = user
@config = config
@gh = gh || GH.with(username: @config.client_id, password: @config.client_secret)
end
def run
token = new_token
@user.update_attributes!(github_oauth_token: token) if token
end
private
def new_token
@new_token ||= @gh.post("/applications/#{client_id}/tokens/#{@user.github_oauth_token}", {})["token"]
end
def client_id
@config.client_id
end
def client_secret
@config.client_secret
end
end
end
end
end
end

View File

@ -1,90 +0,0 @@
require 'gh'
require 'travis/github/education'
module Travis
module Github
module Services
class SyncUser < Travis::Services::Base
class UserInfo
attr_reader :user, :gh
def initialize(user, gh = Github.authenticated(user))
@user, @gh = user, gh
end
def run
if user.github_id != user_info['id'].to_i
raise "Updating #<User id=#{user.id} login=\"#{user.login}\" github_id=#{user.github_id}> failed, github_id differs. github_id on user: #{user.github_id}, github_id from data: #{user_info['id']}"
end
if user.login != login
Travis.logger.info("Changing #<User id=#{user.id} login=\"#{user.login}\" github_id=#{user.github_id}> login: current=\"#{user.login}\", new=\"#{login}\" (UserInfo), data: #{user_info.inspect}")
end
if user.email != email
Travis.logger.info("Changing #<User id=#{user.id} login=\"#{user.login}\" github_id=#{user.github_id}> email: current=\"#{user.email}\", new=\"#{email}\" (UserInfo)")
end
user.update_attributes!(name: name, login: login, gravatar_id: gravatar_id, email: email, education: education)
emails = verified_emails
emails << email unless emails.include? email
emails.each { |e| user.emails.find_or_create_by_email!(e) }
end
def education
if Travis::Features.feature_active?(:education_data_sync) || Travis::Features.owner_active?(:education_data_sync, user)
Education.new(user.github_oauth_token).student?
end
end
def name
user_info['name']
end
def login
user_info.fetch('login')
end
def gravatar_id
user_info['gravatar_id']
end
def email
user_info['email'].presence || primary_email || verified_email || user.email.presence || first_email
end
def verified_emails
emails.select { |e| e["verified"] }.map { |e| e['email'] }
end
private
def emails
return [] unless user.github_scopes.include? 'user' or user.github_scopes.include? 'user:email'
@emails ||= gh['user/emails'].to_a
end
def first_email
emails.first.try(:[], 'email')
end
def primary_email
emails.detect { |e| e["primary"] }.try(:[], 'email')
end
def verified_email
verified_emails.first
end
def user_info
@user_info ||= begin
data = gh['user'].to_hash
if user.login != data['login']
Travis.logger.info("Fetching data for github_id=#{user.github_id} (UserInfo), data: #{data.inspect}")
end
data
end
end
end
end
end
end
end

View File

@ -1,26 +0,0 @@
require 'action_mailer'
require 'i18n'
module Travis
module Mailer
class << self
def config
config = Travis.config.smtp
config ? config.to_h : {}
end
def setup
if config.present?
mailer = ActionMailer::Base
mailer[:delivery_method] = :smtp
mailer[:smtp_settings] = config
@setup = true
end
end
def setup?
!!@setup
end
end
end
end

View File

@ -1,18 +0,0 @@
require 'action_mailer'
class UserMailer < ActionMailer::Base
ActionMailer::Base.append_view_path("#{File.dirname(__FILE__)}/views")
layout 'contact_email'
def welcome_email(user)
@user = user
mail(subject: "Welcome to Travis CI!", from: from, to: user.email) do |format|
format.html
end
end
def from
Travis.config.email.from
end
end

View File

@ -1,170 +0,0 @@
<html style="position: relative;margin: 0;padding: 0;width: 800px;font-family: &quot;Helvetica Neue&quot;, Helvetica, Arial, sans-serif;font-size: 16px;background-color: #fff;">
<head>
<meta content="text/html; charset=UTF-8" http-equiv="Content-Type">
<style type="text/css">
html, body {
position: relative;
margin: 0;
padding: 0;
width: 800px;
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 16px;
background-color: #fff;
}
body {
padding: 20px;
}
h3 {
font-size: 16px;
}
em {
font-style: normal;
background-color: #fff780;
}
dl {
display: inline-block;
margin: 0;
line-height: 24px;
font-size: 15px;
}
dt {
float: left;
clear: left;
width: 85px;
}
dd {
float: left;
}
p sup {
position: relative;
top: -0.5em;
vertical-align: baseline;
line-height: 0;
font-size: 75%;
}
.header {
height: 70px;
}
.header img {
float: left;
width: 200px;
height: 64px;
}
.header ul {
float: right;
margin: 14px 0 0 0;
padding: 0;
font-size: 14px;
line-height: 160%;
color: #999;
list-style-type: none;
}
.header ul a {
color: #999;
text-decoration: none;
}
.header ul a:hover {
text-decoration: underline;
}
.header li {
margin: 0;
padding: 0;
}
.footnotes {
margin: 0;
padding: 0;
font-size: 14px;
color: #999;
list-style-type: none;
}
.footnotes li {
position: relative;
margin: 0;
padding-left: 10px;
line-height: 160%;
}
.footnotes li sup {
position: absolute;
left: 0;
top: -0.33em;
}
.footnotes li a {
color: inherit;
}
p.footnotes {
margin-top: 20px;
}
#invoice .stamp {
position: absolute;
right: 96px;
}
div.footer {
margin-top: 20px;
font-size: 12px;
}
div.footer h2 {
margin-top: 0;
font-size: 18px;
font-weight: bold;
}
div.footer ul {
display: inline-block;
float: left;
margin: 0 35px 0 0;
padding: 0;
list-style-type: none;
}
div.footer li {
line-height: 160%;
font-size: 15px;
color: #999;
margin: 0;
padding: 0;
}
div.footer a {
color: #89a;
}
</style>
</head>
<body style="position: relative;margin: 0;padding: 20px;width: 800px;font-family: &quot;Helvetica Neue&quot;, Helvetica, Arial, sans-serif;font-size: 16px;background-color: #fff;">
<div class="header" style="height: 70px;">
<img src="http://love.travis-ci.com/images/travis-ci.png" style="float: left;width: 200px;height: 64px;">
<ul class="contact" style="float: right;margin: 14px 0 0 0;padding: 0;font-size: 14px;line-height: 160%;color: #999;list-style-type: none;">
<li style="margin: 0;padding: 0;">
E-Mail:
<a href="mailto:support@travis-ci.com" style="color: #999;text-decoration: none;">support@travis-ci.com</a>
</li>
<li style="margin: 0;padding: 0;">
Twitter:
<a href="http://twitter.com/travisci" style="color: #999;text-decoration: none;">@travisci</a>
</li>
</ul>
</div>
<div class="content">
<%= yield %>
</div>
<div class="footer" style="margin-top: 20px;font-size: 12px;">
<ul class="address" style="display: inline-block;float: left;margin: 0 35px 0 0;padding: 0;list-style-type: none;">
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;">Travis CI GmbH</li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;">Rigaerstrasse 8</li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;">10247 Berlin</li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;">Germany</li>
</ul>
<ul style="display: inline-block;float: left;margin: 0 35px 0 0;padding: 0;list-style-type: none;">
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;"><a href="http://travis-ci.com" style="color: #89a;">http://travis-ci.com</a></li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;"><a href="mailto:support@travis-ci.com" style="color: #89a;">support@travis-ci.com</a></li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;"><a href="http://twitter.com/travisci" style="color: #89a;">@travisci</a></li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;"><a href="irc://irc.freenode.net#travis" style="color: #89a;">irc.freenode.net#travis</a></li>
<li style="line-height: 160%;font-size: 15px;color: #999;margin: 0;padding: 0;"><a href="http://chat.travis-ci.com" style="color: #89a;">Support Chat</a></li>
</ul>
</div>
</body>
</html>

View File

@ -1,54 +0,0 @@
<div>
<h3>Welcome to Travis CI!</h3>
<p>
Hey <%= @user.name.blank? ? @user.login : @user.name %>,
</p>
<p>
We'd like to extend a warm welcome to you and provide with some links to help you get started.
</p>
<p>
If you haven't set up your first project yet, head to <a href="https://<%= Travis.config.host %>/profile">your
accounts page</a> and enable the project you'd like to test on Travis CI. Push some code, and your repository
will appear on <a href="https://<%= Travis.config.host %>"><%= Travis.config.host -%></a>.
</p>
<p>
Are you part of an organization that's already running builds on Travis CI? Great news, we've just finished
synchronizing your permissions from GitHub. You can see the projects you have access to and that have already
been built on Travis CI at <a href="https://<%= Travis.config.host %>"><%= Travis.config.host %></a> so you can
dive in right away.
</p>
<p>
Remember to add a .travis.yml file to your project to tell us what steps we should execute to set up your build
environment and run your build. We have sensible defaults, but you're free to customize everything to your
liking.
</p>
<p>
You can find all details on how to <a href="http://docs.travis-ci.com">setup specific languages</a>, the
available <a href="http://docs.travis-ci.com/user/build-configuration/">configuration options</a> for your
builds, and our <a href="http://docs.travis-ci.com/user/ci-environment/">build environment in <a
href="http://docs.travis-ci.com">our documentation</a>. Don't forget to setup <a href="http://docs.travis-ci.com/user/notifications/">notifications</a> if
you'd like to be kept up-to-date about your builds on <a href="http://docs.travis-ci.com/user/notifications/#Campfire-notification">Campfire</a>, <a href="http://docs.travis-ci.com/user/notifications/#HipChat-notification">HipChat</a>, <a href="http://docs.travis-ci.com/user/notifications/#IRC-notification">IRC</a>, and others.
</p>
<p>
If you have any questions or issues, shoot us <a href="mailto:support@travis-ci.com">an email</a>.
</p>
<p>
Have an awesome day!
</p>
<p>
Cheers,
</p>
<p>
The Travis CI Team
</p>
</div>