From 4b6c1773111a43448589d75acff16611d2d1a6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9e=20Hendricksen?= Date: Mon, 30 May 2016 14:30:29 +0200 Subject: [PATCH] moved travis-core to vendor --- .gitignore | 3 +- Gemfile.lock | 4 +- travis-api.gemspec | 18 + vendor/travis-core/lib/travis.rb | 99 +++++ vendor/travis-core/lib/travis/README.markdown | 16 + vendor/travis-core/lib/travis/addons.rb | 31 ++ .../lib/travis/addons/README.markdown | 17 + .../travis-core/lib/travis/addons/archive.rb | 8 + .../travis/addons/archive/event_handler.rb | 36 ++ .../lib/travis/addons/archive/task.rb | 24 ++ .../travis-core/lib/travis/addons/campfire.rb | 13 + .../travis/addons/campfire/event_handler.rb | 33 ++ .../lib/travis/addons/campfire/instruments.rb | 30 ++ vendor/travis-core/lib/travis/addons/email.rb | 10 + .../lib/travis/addons/email/event_handler.rb | 56 +++ .../lib/travis/addons/email/instruments.rb | 30 ++ .../travis-core/lib/travis/addons/flowdock.rb | 10 + .../travis/addons/flowdock/event_handler.rb | 39 ++ .../lib/travis/addons/flowdock/instruments.rb | 31 ++ .../lib/travis/addons/github_status.rb | 10 + .../addons/github_status/event_handler.rb | 74 ++++ .../addons/github_status/instruments.rb | 30 ++ .../travis-core/lib/travis/addons/hipchat.rb | 10 + .../travis/addons/hipchat/event_handler.rb | 40 ++ .../lib/travis/addons/hipchat/instruments.rb | 31 ++ vendor/travis-core/lib/travis/addons/irc.rb | 10 + .../lib/travis/addons/irc/event_handler.rb | 32 ++ .../lib/travis/addons/irc/instruments.rb | 31 ++ .../travis-core/lib/travis/addons/pusher.rb | 11 + .../lib/travis/addons/pusher/event_handler.rb | 57 +++ .../lib/travis/addons/pusher/instruments.rb | 42 +++ .../travis-core/lib/travis/addons/pushover.rb | 13 + .../travis/addons/pushover/event_handler.rb | 37 ++ .../lib/travis/addons/pushover/instruments.rb | 30 ++ vendor/travis-core/lib/travis/addons/slack.rb | 11 + .../lib/travis/addons/slack/event_handler.rb | 30 ++ .../lib/travis/addons/slack/instruments.rb | 25 ++ .../travis-core/lib/travis/addons/sqwiggle.rb | 11 + .../travis/addons/sqwiggle/event_handler.rb | 30 ++ .../lib/travis/addons/sqwiggle/instruments.rb | 29 ++ .../lib/travis/addons/states_cache.rb | 7 + .../addons/states_cache/event_handler.rb | 47 +++ vendor/travis-core/lib/travis/addons/util.rb | 7 + .../travis-core/lib/travis/addons/webhook.rb | 11 + .../travis/addons/webhook/event_handler.rb | 39 ++ .../lib/travis/addons/webhook/instruments.rb | 29 ++ .../travis-core/lib/travis/advisory_locks.rb | 59 +++ vendor/travis-core/lib/travis/api.rb | 67 ++++ .../lib/travis/api/README.markdown | 10 + vendor/travis-core/lib/travis/api/formats.rb | 9 + vendor/travis-core/lib/travis/api/v0.rb | 11 + vendor/travis-core/lib/travis/api/v0/event.rb | 12 + .../lib/travis/api/v0/event/build.rb | 95 +++++ .../lib/travis/api/v0/event/job.rb | 38 ++ .../lib/travis/api/v0/notification.rb | 12 + .../lib/travis/api/v0/notification/build.rb | 28 ++ .../travis/api/v0/notification/repository.rb | 28 ++ .../lib/travis/api/v0/notification/user.rb | 28 ++ .../travis-core/lib/travis/api/v0/pusher.rb | 11 + .../lib/travis/api/v0/pusher/annotation.rb | 33 ++ .../api/v0/pusher/annotation/created.rb | 12 + .../api/v0/pusher/annotation/updated.rb | 12 + .../lib/travis/api/v0/pusher/build.rb | 111 ++++++ .../travis/api/v0/pusher/build/canceled.rb | 12 + .../lib/travis/api/v0/pusher/build/created.rb | 15 + .../travis/api/v0/pusher/build/finished.rb | 14 + .../travis/api/v0/pusher/build/received.rb | 12 + .../api/v0/pusher/build/received/job.rb | 47 +++ .../lib/travis/api/v0/pusher/build/started.rb | 13 + .../travis/api/v0/pusher/build/started/job.rb | 47 +++ .../lib/travis/api/v0/pusher/job.rb | 67 ++++ .../lib/travis/api/v0/pusher/job/canceled.rb | 12 + .../lib/travis/api/v0/pusher/job/created.rb | 12 + .../lib/travis/api/v0/pusher/job/finished.rb | 12 + .../lib/travis/api/v0/pusher/job/log.rb | 31 ++ .../lib/travis/api/v0/pusher/job/received.rb | 12 + .../lib/travis/api/v0/pusher/job/started.rb | 12 + .../travis-core/lib/travis/api/v0/worker.rb | 9 + .../lib/travis/api/v0/worker/job.rb | 33 ++ .../lib/travis/api/v0/worker/job/test.rb | 118 ++++++ vendor/travis-core/lib/travis/api/v1.rb | 10 + .../travis-core/lib/travis/api/v1/archive.rb | 9 + .../lib/travis/api/v1/archive/build.rb | 50 +++ .../lib/travis/api/v1/archive/build/job.rb | 31 ++ .../travis-core/lib/travis/api/v1/helpers.rb | 9 + .../lib/travis/api/v1/helpers/legacy.rb | 34 ++ vendor/travis-core/lib/travis/api/v1/http.rb | 17 + .../lib/travis/api/v1/http/branches.rb | 43 +++ .../lib/travis/api/v1/http/build.rb | 47 +++ .../lib/travis/api/v1/http/build/job.rb | 32 ++ .../lib/travis/api/v1/http/builds.rb | 38 ++ .../lib/travis/api/v1/http/hooks.rb | 34 ++ .../travis-core/lib/travis/api/v1/http/job.rb | 44 +++ .../lib/travis/api/v1/http/jobs.rb | 34 ++ .../lib/travis/api/v1/http/repositories.rb | 37 ++ .../lib/travis/api/v1/http/repository.rb | 34 ++ .../lib/travis/api/v1/http/user.rb | 31 ++ .../travis-core/lib/travis/api/v1/webhook.rb | 9 + .../lib/travis/api/v1/webhook/build.rb | 27 ++ .../travis/api/v1/webhook/build/finished.rb | 70 ++++ .../api/v1/webhook/build/finished/job.rb | 50 +++ vendor/travis-core/lib/travis/api/v2.rb | 7 + vendor/travis-core/lib/travis/chunkifier.rb | 59 +++ .../travis-core/lib/travis/commit_command.rb | 23 ++ .../travis-core/lib/travis/config/database.rb | 40 ++ .../travis-core/lib/travis/config/defaults.rb | 72 ++++ vendor/travis-core/lib/travis/config/url.rb | 36 ++ vendor/travis-core/lib/travis/engine.rb | 21 ++ vendor/travis-core/lib/travis/enqueue.rb | 6 + .../lib/travis/enqueue/services.rb | 14 + .../travis/enqueue/services/enqueue_jobs.rb | 127 +++++++ .../enqueue/services/enqueue_jobs/limit.rb | 85 +++++ vendor/travis-core/lib/travis/errors.rb | 20 + vendor/travis-core/lib/travis/event.rb | 57 +++ vendor/travis-core/lib/travis/event/config.rb | 109 ++++++ .../travis-core/lib/travis/event/handler.rb | 81 ++++ .../lib/travis/event/handler/metrics.rb | 50 +++ .../lib/travis/event/handler/trail.rb | 42 +++ .../lib/travis/event/subscription.rb | 61 +++ vendor/travis-core/lib/travis/features.rb | 146 +++++++ vendor/travis-core/lib/travis/github.rb | 36 ++ .../lib/travis/github/education.rb | 45 +++ .../travis-core/lib/travis/github/services.rb | 18 + .../travis/github/services/fetch_config.rb | 90 +++++ .../github/services/find_or_create_org.rb | 80 ++++ .../github/services/find_or_create_repo.rb | 44 +++ .../github/services/find_or_create_user.rb | 66 ++++ .../lib/travis/github/services/set_hook.rb | 67 ++++ .../lib/travis/github/services/sync_user.rb | 74 ++++ .../services/sync_user/organizations.rb | 143 +++++++ .../github/services/sync_user/repositories.rb | 140 +++++++ .../github/services/sync_user/repository.rb | 140 +++++++ .../github/services/sync_user/reset_token.rb | 36 ++ .../github/services/sync_user/user_info.rb | 90 +++++ vendor/travis-core/lib/travis/logs.rb | 6 + .../travis-core/lib/travis/logs/services.rb | 17 + .../lib/travis/logs/services/aggregate.rb | 96 +++++ .../lib/travis/logs/services/archive.rb | 167 ++++++++ .../lib/travis/logs/services/receive.rb | 79 ++++ vendor/travis-core/lib/travis/mailer.rb | 26 ++ .../lib/travis/mailer/user_mailer.rb | 18 + .../views/layouts/contact_email.html.erb | 170 +++++++++ .../views/user_mailer/welcome_email.html.erb | 54 +++ vendor/travis-core/lib/travis/model.rb | 60 +++ .../travis-core/lib/travis/model/account.rb | 22 ++ .../lib/travis/model/annotation.rb | 28 ++ .../lib/travis/model/annotation_provider.rb | 20 + vendor/travis-core/lib/travis/model/branch.rb | 6 + .../travis-core/lib/travis/model/broadcast.rb | 30 ++ vendor/travis-core/lib/travis/model/build.rb | 228 +++++++++++ .../lib/travis/model/build/config.rb | 119 ++++++ .../lib/travis/model/build/config/dist.rb | 59 +++ .../lib/travis/model/build/config/env.rb | 50 +++ .../lib/travis/model/build/config/features.rb | 22 ++ .../lib/travis/model/build/config/group.rb | 18 + .../lib/travis/model/build/config/language.rb | 27 ++ .../lib/travis/model/build/config/matrix.rb | 108 ++++++ .../travis/model/build/config/obfuscate.rb | 53 +++ .../lib/travis/model/build/config/os.rb | 27 ++ .../lib/travis/model/build/config/yaml.rb | 14 + .../lib/travis/model/build/denormalize.rb | 34 ++ .../lib/travis/model/build/matrix.rb | 114 ++++++ .../lib/travis/model/build/metrics.rb | 15 + .../lib/travis/model/build/result_message.rb | 79 ++++ .../lib/travis/model/build/states.rb | 109 ++++++ .../lib/travis/model/build/update_branch.rb | 37 ++ vendor/travis-core/lib/travis/model/commit.rb | 28 ++ vendor/travis-core/lib/travis/model/email.rb | 5 + .../lib/travis/model/encrypted_column.rb | 105 ++++++ .../lib/travis/model/env_helpers.rb | 10 + vendor/travis-core/lib/travis/model/job.rb | 235 ++++++++++++ .../lib/travis/model/job/cleanup.rb | 41 ++ .../travis-core/lib/travis/model/job/queue.rb | 121 ++++++ .../travis-core/lib/travis/model/job/test.rb | 107 ++++++ vendor/travis-core/lib/travis/model/log.rb | 54 +++ .../lib/travis/model/logs_model.rb | 3 + .../lib/travis/model/membership.rb | 7 + .../lib/travis/model/organization.rb | 14 + .../lib/travis/model/permission.rb | 26 ++ .../lib/travis/model/repository.rb | 206 ++++++++++ .../lib/travis/model/repository/settings.rb | 135 +++++++ .../travis/model/repository/status_image.rb | 61 +++ .../travis-core/lib/travis/model/request.rb | 115 ++++++ .../lib/travis/model/request/approval.rb | 130 +++++++ .../lib/travis/model/request/branches.rb | 57 +++ .../lib/travis/model/request/pull_request.rb | 39 ++ .../lib/travis/model/request/states.rb | 119 ++++++ .../travis-core/lib/travis/model/ssl_key.rb | 72 ++++ vendor/travis-core/lib/travis/model/token.rb | 25 ++ vendor/travis-core/lib/travis/model/url.rb | 23 ++ vendor/travis-core/lib/travis/model/user.rb | 150 ++++++++ .../lib/travis/model/user/oauth.rb | 28 ++ .../lib/travis/model/user/renaming.rb | 21 ++ vendor/travis-core/lib/travis/notification.rb | 24 ++ .../lib/travis/notification/instrument.rb | 58 +++ .../notification/instrument/event_handler.rb | 35 ++ .../travis/notification/instrument/task.rb | 34 ++ .../lib/travis/notification/publisher.rb | 9 + .../lib/travis/notification/publisher/log.rb | 42 +++ .../travis/notification/publisher/memory.rb | 15 + .../travis/notification/publisher/redis.rb | 50 +++ .../travis/overwritable_method_definitions.rb | 39 ++ vendor/travis-core/lib/travis/redis_pool.rb | 32 ++ vendor/travis-core/lib/travis/requests.rb | 6 + .../lib/travis/requests/services.rb | 13 + .../lib/travis/requests/services/receive.rb | 207 ++++++++++ .../travis/requests/services/receive/api.rb | 106 ++++++ .../travis/requests/services/receive/cron.rb | 10 + .../requests/services/receive/pull_request.rb | 117 ++++++ .../travis/requests/services/receive/push.rb | 84 +++++ .../travis-core/lib/travis/secure_config.rb | 71 ++++ vendor/travis-core/lib/travis/services.rb | 74 ++++ .../travis-core/lib/travis/services/base.rb | 28 ++ .../lib/travis/services/cancel_build.rb | 78 ++++ .../lib/travis/services/cancel_job.rb | 73 ++++ .../lib/travis/services/delete_caches.rb | 15 + .../lib/travis/services/find_admin.rb | 94 +++++ .../lib/travis/services/find_annotations.rb | 17 + .../lib/travis/services/find_branch.rb | 36 ++ .../lib/travis/services/find_branches.rb | 32 ++ .../lib/travis/services/find_build.rb | 55 +++ .../lib/travis/services/find_builds.rb | 54 +++ .../lib/travis/services/find_caches.rb | 191 ++++++++++ .../travis/services/find_daily_repos_stats.rb | 23 ++ .../travis/services/find_daily_tests_stats.rb | 23 ++ .../lib/travis/services/find_hooks.rb | 13 + .../lib/travis/services/find_job.rb | 48 +++ .../lib/travis/services/find_jobs.rb | 43 +++ .../lib/travis/services/find_log.rb | 33 ++ .../lib/travis/services/find_repo.rb | 23 ++ .../lib/travis/services/find_repo_key.rb | 23 ++ .../lib/travis/services/find_repo_settings.rb | 31 ++ .../lib/travis/services/find_repos.rb | 53 +++ .../lib/travis/services/find_request.rb | 25 ++ .../lib/travis/services/find_requests.rb | 50 +++ .../lib/travis/services/find_user_accounts.rb | 33 ++ .../travis/services/find_user_broadcasts.rb | 13 + .../travis/services/find_user_permissions.rb | 15 + .../lib/travis/services/helpers.rb | 20 + .../lib/travis/services/next_build_number.rb | 40 ++ .../travis/services/regenerate_repo_key.rb | 35 ++ .../lib/travis/services/registry.rb | 31 ++ .../lib/travis/services/remove_log.rb | 79 ++++ .../lib/travis/services/reset_model.rb | 74 ++++ .../lib/travis/services/sync_user.rb | 26 ++ .../lib/travis/services/update_annotation.rb | 36 ++ .../lib/travis/services/update_hook.rb | 47 +++ .../lib/travis/services/update_job.rb | 71 ++++ .../lib/travis/services/update_log.rb | 31 ++ .../lib/travis/services/update_user.rb | 38 ++ vendor/travis-core/lib/travis/settings.rb | 67 ++++ .../lib/travis/settings/collection.rb | 79 ++++ .../lib/travis/settings/encrypted_value.rb | 62 +++ .../travis-core/lib/travis/settings/model.rb | 39 ++ .../lib/travis/settings/model_extensions.rb | 177 +++++++++ vendor/travis-core/lib/travis/states_cache.rb | 158 ++++++++ vendor/travis-core/lib/travis/task.rb | 93 +++++ vendor/travis-core/lib/travis/testing.rb | 9 + .../lib/travis/testing/factories.rb | 135 +++++++ .../lib/travis/testing/matchers.rb | 50 +++ .../lib/travis/testing/payloads.rb | 248 ++++++++++++ .../lib/travis/testing/scenario.rb | 115 ++++++ .../travis-core/lib/travis/testing/stubs.rb | 356 ++++++++++++++++++ .../lib/travis/testing/stubs/stub.rb | 45 +++ .../lib/travis/travis_yml_stats.rb | 195 ++++++++++ vendor/travis-core/lib/travis_core/version.rb | 3 + vendor/travis-core/travis-core.gemspec | 49 +++ 267 files changed, 13512 insertions(+), 4 deletions(-) create mode 100644 vendor/travis-core/lib/travis.rb create mode 100644 vendor/travis-core/lib/travis/README.markdown create mode 100644 vendor/travis-core/lib/travis/addons.rb create mode 100644 vendor/travis-core/lib/travis/addons/README.markdown create mode 100644 vendor/travis-core/lib/travis/addons/archive.rb create mode 100644 vendor/travis-core/lib/travis/addons/archive/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/archive/task.rb create mode 100644 vendor/travis-core/lib/travis/addons/campfire.rb create mode 100644 vendor/travis-core/lib/travis/addons/campfire/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/campfire/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/email.rb create mode 100644 vendor/travis-core/lib/travis/addons/email/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/email/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/flowdock.rb create mode 100644 vendor/travis-core/lib/travis/addons/flowdock/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/flowdock/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/github_status.rb create mode 100644 vendor/travis-core/lib/travis/addons/github_status/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/github_status/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/hipchat.rb create mode 100644 vendor/travis-core/lib/travis/addons/hipchat/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/hipchat/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/irc.rb create mode 100644 vendor/travis-core/lib/travis/addons/irc/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/irc/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/pusher.rb create mode 100644 vendor/travis-core/lib/travis/addons/pusher/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/pusher/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/pushover.rb create mode 100644 vendor/travis-core/lib/travis/addons/pushover/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/pushover/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/slack.rb create mode 100644 vendor/travis-core/lib/travis/addons/slack/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/slack/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/sqwiggle.rb create mode 100644 vendor/travis-core/lib/travis/addons/sqwiggle/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/sqwiggle/instruments.rb create mode 100644 vendor/travis-core/lib/travis/addons/states_cache.rb create mode 100644 vendor/travis-core/lib/travis/addons/states_cache/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/util.rb create mode 100644 vendor/travis-core/lib/travis/addons/webhook.rb create mode 100644 vendor/travis-core/lib/travis/addons/webhook/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/addons/webhook/instruments.rb create mode 100644 vendor/travis-core/lib/travis/advisory_locks.rb create mode 100644 vendor/travis-core/lib/travis/api.rb create mode 100644 vendor/travis-core/lib/travis/api/README.markdown create mode 100644 vendor/travis-core/lib/travis/api/formats.rb create mode 100644 vendor/travis-core/lib/travis/api/v0.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/event.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/event/build.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/event/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/notification.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/notification/build.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/notification/repository.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/notification/user.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/annotation.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/annotation/created.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/annotation/updated.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/canceled.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/created.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/finished.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/received.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/received/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/started.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/build/started/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job/canceled.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job/created.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job/finished.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job/log.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job/received.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/pusher/job/started.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/worker.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/worker/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v0/worker/job/test.rb create mode 100644 vendor/travis-core/lib/travis/api/v1.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/archive.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/archive/build.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/archive/build/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/helpers.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/helpers/legacy.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/branches.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/build.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/build/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/builds.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/hooks.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/jobs.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/repositories.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/repository.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/http/user.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/webhook.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/webhook/build.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/webhook/build/finished.rb create mode 100644 vendor/travis-core/lib/travis/api/v1/webhook/build/finished/job.rb create mode 100644 vendor/travis-core/lib/travis/api/v2.rb create mode 100644 vendor/travis-core/lib/travis/chunkifier.rb create mode 100644 vendor/travis-core/lib/travis/commit_command.rb create mode 100644 vendor/travis-core/lib/travis/config/database.rb create mode 100644 vendor/travis-core/lib/travis/config/defaults.rb create mode 100644 vendor/travis-core/lib/travis/config/url.rb create mode 100644 vendor/travis-core/lib/travis/engine.rb create mode 100644 vendor/travis-core/lib/travis/enqueue.rb create mode 100644 vendor/travis-core/lib/travis/enqueue/services.rb create mode 100644 vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb create mode 100644 vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb create mode 100644 vendor/travis-core/lib/travis/errors.rb create mode 100644 vendor/travis-core/lib/travis/event.rb create mode 100644 vendor/travis-core/lib/travis/event/config.rb create mode 100644 vendor/travis-core/lib/travis/event/handler.rb create mode 100644 vendor/travis-core/lib/travis/event/handler/metrics.rb create mode 100644 vendor/travis-core/lib/travis/event/handler/trail.rb create mode 100644 vendor/travis-core/lib/travis/event/subscription.rb create mode 100644 vendor/travis-core/lib/travis/features.rb create mode 100644 vendor/travis-core/lib/travis/github.rb create mode 100644 vendor/travis-core/lib/travis/github/education.rb create mode 100644 vendor/travis-core/lib/travis/github/services.rb create mode 100644 vendor/travis-core/lib/travis/github/services/fetch_config.rb create mode 100644 vendor/travis-core/lib/travis/github/services/find_or_create_org.rb create mode 100644 vendor/travis-core/lib/travis/github/services/find_or_create_repo.rb create mode 100644 vendor/travis-core/lib/travis/github/services/find_or_create_user.rb create mode 100644 vendor/travis-core/lib/travis/github/services/set_hook.rb create mode 100644 vendor/travis-core/lib/travis/github/services/sync_user.rb create mode 100644 vendor/travis-core/lib/travis/github/services/sync_user/organizations.rb create mode 100644 vendor/travis-core/lib/travis/github/services/sync_user/repositories.rb create mode 100644 vendor/travis-core/lib/travis/github/services/sync_user/repository.rb create mode 100644 vendor/travis-core/lib/travis/github/services/sync_user/reset_token.rb create mode 100644 vendor/travis-core/lib/travis/github/services/sync_user/user_info.rb create mode 100644 vendor/travis-core/lib/travis/logs.rb create mode 100644 vendor/travis-core/lib/travis/logs/services.rb create mode 100644 vendor/travis-core/lib/travis/logs/services/aggregate.rb create mode 100644 vendor/travis-core/lib/travis/logs/services/archive.rb create mode 100644 vendor/travis-core/lib/travis/logs/services/receive.rb create mode 100644 vendor/travis-core/lib/travis/mailer.rb create mode 100644 vendor/travis-core/lib/travis/mailer/user_mailer.rb create mode 100644 vendor/travis-core/lib/travis/mailer/views/layouts/contact_email.html.erb create mode 100644 vendor/travis-core/lib/travis/mailer/views/user_mailer/welcome_email.html.erb create mode 100644 vendor/travis-core/lib/travis/model.rb create mode 100644 vendor/travis-core/lib/travis/model/account.rb create mode 100644 vendor/travis-core/lib/travis/model/annotation.rb create mode 100644 vendor/travis-core/lib/travis/model/annotation_provider.rb create mode 100644 vendor/travis-core/lib/travis/model/branch.rb create mode 100644 vendor/travis-core/lib/travis/model/broadcast.rb create mode 100644 vendor/travis-core/lib/travis/model/build.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/dist.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/env.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/features.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/group.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/language.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/matrix.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/obfuscate.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/os.rb create mode 100644 vendor/travis-core/lib/travis/model/build/config/yaml.rb create mode 100644 vendor/travis-core/lib/travis/model/build/denormalize.rb create mode 100644 vendor/travis-core/lib/travis/model/build/matrix.rb create mode 100644 vendor/travis-core/lib/travis/model/build/metrics.rb create mode 100644 vendor/travis-core/lib/travis/model/build/result_message.rb create mode 100644 vendor/travis-core/lib/travis/model/build/states.rb create mode 100644 vendor/travis-core/lib/travis/model/build/update_branch.rb create mode 100644 vendor/travis-core/lib/travis/model/commit.rb create mode 100644 vendor/travis-core/lib/travis/model/email.rb create mode 100644 vendor/travis-core/lib/travis/model/encrypted_column.rb create mode 100644 vendor/travis-core/lib/travis/model/env_helpers.rb create mode 100644 vendor/travis-core/lib/travis/model/job.rb create mode 100644 vendor/travis-core/lib/travis/model/job/cleanup.rb create mode 100644 vendor/travis-core/lib/travis/model/job/queue.rb create mode 100644 vendor/travis-core/lib/travis/model/job/test.rb create mode 100644 vendor/travis-core/lib/travis/model/log.rb create mode 100644 vendor/travis-core/lib/travis/model/logs_model.rb create mode 100644 vendor/travis-core/lib/travis/model/membership.rb create mode 100644 vendor/travis-core/lib/travis/model/organization.rb create mode 100644 vendor/travis-core/lib/travis/model/permission.rb create mode 100644 vendor/travis-core/lib/travis/model/repository.rb create mode 100644 vendor/travis-core/lib/travis/model/repository/settings.rb create mode 100644 vendor/travis-core/lib/travis/model/repository/status_image.rb create mode 100644 vendor/travis-core/lib/travis/model/request.rb create mode 100644 vendor/travis-core/lib/travis/model/request/approval.rb create mode 100644 vendor/travis-core/lib/travis/model/request/branches.rb create mode 100644 vendor/travis-core/lib/travis/model/request/pull_request.rb create mode 100644 vendor/travis-core/lib/travis/model/request/states.rb create mode 100644 vendor/travis-core/lib/travis/model/ssl_key.rb create mode 100644 vendor/travis-core/lib/travis/model/token.rb create mode 100644 vendor/travis-core/lib/travis/model/url.rb create mode 100644 vendor/travis-core/lib/travis/model/user.rb create mode 100644 vendor/travis-core/lib/travis/model/user/oauth.rb create mode 100644 vendor/travis-core/lib/travis/model/user/renaming.rb create mode 100644 vendor/travis-core/lib/travis/notification.rb create mode 100644 vendor/travis-core/lib/travis/notification/instrument.rb create mode 100644 vendor/travis-core/lib/travis/notification/instrument/event_handler.rb create mode 100644 vendor/travis-core/lib/travis/notification/instrument/task.rb create mode 100644 vendor/travis-core/lib/travis/notification/publisher.rb create mode 100644 vendor/travis-core/lib/travis/notification/publisher/log.rb create mode 100644 vendor/travis-core/lib/travis/notification/publisher/memory.rb create mode 100644 vendor/travis-core/lib/travis/notification/publisher/redis.rb create mode 100644 vendor/travis-core/lib/travis/overwritable_method_definitions.rb create mode 100644 vendor/travis-core/lib/travis/redis_pool.rb create mode 100644 vendor/travis-core/lib/travis/requests.rb create mode 100644 vendor/travis-core/lib/travis/requests/services.rb create mode 100644 vendor/travis-core/lib/travis/requests/services/receive.rb create mode 100644 vendor/travis-core/lib/travis/requests/services/receive/api.rb create mode 100644 vendor/travis-core/lib/travis/requests/services/receive/cron.rb create mode 100644 vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb create mode 100644 vendor/travis-core/lib/travis/requests/services/receive/push.rb create mode 100644 vendor/travis-core/lib/travis/secure_config.rb create mode 100644 vendor/travis-core/lib/travis/services.rb create mode 100644 vendor/travis-core/lib/travis/services/base.rb create mode 100644 vendor/travis-core/lib/travis/services/cancel_build.rb create mode 100644 vendor/travis-core/lib/travis/services/cancel_job.rb create mode 100644 vendor/travis-core/lib/travis/services/delete_caches.rb create mode 100644 vendor/travis-core/lib/travis/services/find_admin.rb create mode 100644 vendor/travis-core/lib/travis/services/find_annotations.rb create mode 100644 vendor/travis-core/lib/travis/services/find_branch.rb create mode 100644 vendor/travis-core/lib/travis/services/find_branches.rb create mode 100644 vendor/travis-core/lib/travis/services/find_build.rb create mode 100644 vendor/travis-core/lib/travis/services/find_builds.rb create mode 100644 vendor/travis-core/lib/travis/services/find_caches.rb create mode 100644 vendor/travis-core/lib/travis/services/find_daily_repos_stats.rb create mode 100644 vendor/travis-core/lib/travis/services/find_daily_tests_stats.rb create mode 100644 vendor/travis-core/lib/travis/services/find_hooks.rb create mode 100644 vendor/travis-core/lib/travis/services/find_job.rb create mode 100644 vendor/travis-core/lib/travis/services/find_jobs.rb create mode 100644 vendor/travis-core/lib/travis/services/find_log.rb create mode 100644 vendor/travis-core/lib/travis/services/find_repo.rb create mode 100644 vendor/travis-core/lib/travis/services/find_repo_key.rb create mode 100644 vendor/travis-core/lib/travis/services/find_repo_settings.rb create mode 100644 vendor/travis-core/lib/travis/services/find_repos.rb create mode 100644 vendor/travis-core/lib/travis/services/find_request.rb create mode 100644 vendor/travis-core/lib/travis/services/find_requests.rb create mode 100644 vendor/travis-core/lib/travis/services/find_user_accounts.rb create mode 100644 vendor/travis-core/lib/travis/services/find_user_broadcasts.rb create mode 100644 vendor/travis-core/lib/travis/services/find_user_permissions.rb create mode 100644 vendor/travis-core/lib/travis/services/helpers.rb create mode 100644 vendor/travis-core/lib/travis/services/next_build_number.rb create mode 100644 vendor/travis-core/lib/travis/services/regenerate_repo_key.rb create mode 100644 vendor/travis-core/lib/travis/services/registry.rb create mode 100644 vendor/travis-core/lib/travis/services/remove_log.rb create mode 100644 vendor/travis-core/lib/travis/services/reset_model.rb create mode 100644 vendor/travis-core/lib/travis/services/sync_user.rb create mode 100644 vendor/travis-core/lib/travis/services/update_annotation.rb create mode 100644 vendor/travis-core/lib/travis/services/update_hook.rb create mode 100644 vendor/travis-core/lib/travis/services/update_job.rb create mode 100644 vendor/travis-core/lib/travis/services/update_log.rb create mode 100644 vendor/travis-core/lib/travis/services/update_user.rb create mode 100644 vendor/travis-core/lib/travis/settings.rb create mode 100644 vendor/travis-core/lib/travis/settings/collection.rb create mode 100644 vendor/travis-core/lib/travis/settings/encrypted_value.rb create mode 100644 vendor/travis-core/lib/travis/settings/model.rb create mode 100644 vendor/travis-core/lib/travis/settings/model_extensions.rb create mode 100644 vendor/travis-core/lib/travis/states_cache.rb create mode 100644 vendor/travis-core/lib/travis/task.rb create mode 100644 vendor/travis-core/lib/travis/testing.rb create mode 100644 vendor/travis-core/lib/travis/testing/factories.rb create mode 100644 vendor/travis-core/lib/travis/testing/matchers.rb create mode 100644 vendor/travis-core/lib/travis/testing/payloads.rb create mode 100644 vendor/travis-core/lib/travis/testing/scenario.rb create mode 100644 vendor/travis-core/lib/travis/testing/stubs.rb create mode 100644 vendor/travis-core/lib/travis/testing/stubs/stub.rb create mode 100644 vendor/travis-core/lib/travis/travis_yml_stats.rb create mode 100644 vendor/travis-core/lib/travis_core/version.rb create mode 100644 vendor/travis-core/travis-core.gemspec diff --git a/.gitignore b/.gitignore index bc1ee12d..5c9279d7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,7 @@ config/skylight.yml tmp/ log/ logs/ - -vendor +!vendor/travis-core/lib/travis/logs/ .yardoc .coverage diff --git a/Gemfile.lock b/Gemfile.lock index 2ddb4d64..3516706b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -49,9 +49,9 @@ GIT GIT remote: git://github.com/travis-ci/travis-migrations.git - revision: bf360857ef7830f7e3ff12de181ab58c33fb29f1 + revision: dc432e45354287c617c3ae07a72e9e3c4be012cd specs: - travis-migrations (0.0.1) + travis-migrations (0.0.2) GIT remote: git://github.com/travis-ci/travis-sidekiqs.git diff --git a/travis-api.gemspec b/travis-api.gemspec index 005870e0..20478e67 100644 --- a/travis-api.gemspec +++ b/travis-api.gemspec @@ -13,6 +13,24 @@ Gem::Specification.new do |s| "Andre Arko", "Brian Ford", "Bryan Goldstein", + "Konstantin Haase", + "Piotr Sarnacki", + "carlad", + "Sven Fuchs", + "Hiro Asari", + "Mathias Meyer", + "Josh Kalderimis", + "Henrik Hodne", + "Steffen Kötte", + "Tyranja", + "Lennard Wolf", + "Renée Hendricksen", + "Ana Rosas", + "Steffen", + "Jonas Chromik", + "Dan Buch", + "Andre Arko", + "Erik Michaels-Ober", "C. Scott Ananian", "Christopher Weyand", "Dan Buch", diff --git a/vendor/travis-core/lib/travis.rb b/vendor/travis-core/lib/travis.rb new file mode 100644 index 00000000..6bb017cd --- /dev/null +++ b/vendor/travis-core/lib/travis.rb @@ -0,0 +1,99 @@ +require 'pusher' +require 'travis/support' +require 'travis/support/database' +require 'travis_core/version' +require 'travis/redis_pool' +require 'travis/errors' + +# travis-core holds the central parts of the model layer used in both travis-ci +# (i.e. the web application) as well as travis-hub (a non-rails ui-less JRuby +# application that receives, processes and distributes messages from/to the +# workers and issues various events like email, pusher, irc notifications and +# so on). +# +# travis/model - contains ActiveRecord models that and model the main +# parts of the domain logic (e.g. repository, build, job +# etc.) and issue events on state changes (e.g. +# build:created, job:test:finished etc.) +# travis/event - contains event handlers that register for certain +# events and send out such things as email, pusher, irc +# notifications, archive builds or queue jobs for the +# workers. +# travis/mailer - contains ActionMailers for sending out email +# notifications +# +# travis-core also contains some helper classes and modules like Travis::Database +# (needed in travis-hub in order to connect to the database) and Travis::Renderer +# (our inferior layer on top of Rabl). +module Travis + class << self + def services=(services) + # Travis.logger.info("Using services: #{services}") + @services = services + end + + def services + @services ||= Travis::Services + end + end + + require 'travis/model' + require 'travis/task' + require 'travis/event' + require 'travis/addons' + require 'travis/api' + require 'travis/config/defaults' + require 'travis/commit_command' + require 'travis/enqueue' + require 'travis/features' + require 'travis/github' + require 'travis/logs' + require 'travis/mailer' + require 'travis/notification' + require 'travis/requests' + require 'travis/services' + + class UnknownRepository < StandardError; end + class GithubApiError < StandardError; end + class AdminMissing < StandardError; end + class RepositoryMissing < StandardError; end + class LogAlreadyRemoved < StandardError; end + class AuthorizationDenied < StandardError; end + class JobUnfinished < StandardError; end + + class << self + def setup(options = {}) + @config = Config.load(*options[:configs]) + @redis = Travis::RedisPool.new(config.redis.to_h) + + Travis.logger.info('Setting up Travis::Core') + + Github.setup + Addons.register + Services.register + Enqueue::Services.register + Github::Services.register + Logs::Services.register + Requests::Services.register + end + + attr_accessor :redis, :config + + def pusher + @pusher ||= ::Pusher.tap do |pusher| + pusher.app_id = config.pusher.app_id + pusher.key = config.pusher.key + pusher.secret = config.pusher.secret + pusher.scheme = config.pusher.scheme if config.pusher.scheme + pusher.host = config.pusher.host if config.pusher.host + pusher.port = config.pusher.port if config.pusher.port + end + end + + def states_cache + @states_cache ||= Travis::StatesCache.new + end + end + + setup +end diff --git a/vendor/travis-core/lib/travis/README.markdown b/vendor/travis-core/lib/travis/README.markdown new file mode 100644 index 00000000..a24a9c90 --- /dev/null +++ b/vendor/travis-core/lib/travis/README.markdown @@ -0,0 +1,16 @@ +# Travis Core directory overview + +This folder, `lib/travis` contains the main code for the Travis Core repository. It contains several sub-section/subdirectories: + +- [`addons`](addons): Event handlers that take events such as "build finished" and sends out notifications to GitHub, Pusher, Campfire, etc. +- [`api`](api): Serializers for models and events used in our API and in some other places (for example to generate Pusher payloads). +- [`enqueue`](enqueue): Logic for enqueueing jobs. +- [`event`](event): Code for sending and subscribing to events. Used by the `addons` code to subscribe to changes in the models. +- [`github`](github): Services for communicating with the GitHub API. +- [`mailer`](mailer): ActionMailer mailers. +- [`model`](model): All of our ActiveRecord models. +- [`notification`](notification): Code for adding instrumentation. +- [`requests`](requests): Handles requests received from GitHub. +- [`secure_config.rb`](secure_config.rb): Logic for encrypting and decrypting build/job configs. +- [`services`](services): Most of the business logic behind our [API](https://github.com/travis-ci/travis-api). +- [`testing`](testing): Code used by our tests, such as model stubs and factories. diff --git a/vendor/travis-core/lib/travis/addons.rb b/vendor/travis-core/lib/travis/addons.rb new file mode 100644 index 00000000..240c66cc --- /dev/null +++ b/vendor/travis-core/lib/travis/addons.rb @@ -0,0 +1,31 @@ +require 'travis/notification' + +module Travis + module Addons + require 'travis/addons/archive' + require 'travis/addons/campfire' + require 'travis/addons/email' + require 'travis/addons/flowdock' + require 'travis/addons/github_status' + require 'travis/addons/hipchat' + require 'travis/addons/irc' + require 'travis/addons/pusher' + require 'travis/addons/states_cache' + require 'travis/addons/sqwiggle' + require 'travis/addons/webhook' + require 'travis/addons/slack' + require 'travis/addons/pushover' + + class << self + def register + constants(false).each do |name| + key = name.to_s.underscore + const = const_get(name) + handler = const.const_get(:EventHandler) rescue nil + Travis::Event::Subscription.register(key, handler) if handler + const.setup if const.respond_to?(:setup) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/README.markdown b/vendor/travis-core/lib/travis/addons/README.markdown new file mode 100644 index 00000000..936031d5 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/README.markdown @@ -0,0 +1,17 @@ +# Travis Core Addons + +The Addons are event handlers that accepts events such as "build finished" and forwards them to different services. The different services are: + +- Campfire +- E-mail +- Flowdock +- GitHub Commit Statuses +- Hipchat +- IRC +- Pusher: Used to update our Web UI automatically. +- Sqwiggle +- States cache: Caches the state of each branch in Memcached for status images. +- Webhook +- Pushover + +To add a new notification service, an event handler and a task is needed. The event handler is run by [`travis-hub`](https://github.com/travis-ci/travis-hub) and has access to the database. This should check whether the event should be forwarded at all, and pull out any necessary configuration values. It should then asynchronously run the corresponding Task. The Task is run by [`travis-tasks`](https://github.com/travis-ci/travis-tasks) via Sidekiq and should do the actual API calls needed. The event handler should finish very quickly, while the task is allowed to take longer. diff --git a/vendor/travis-core/lib/travis/addons/archive.rb b/vendor/travis-core/lib/travis/addons/archive.rb new file mode 100644 index 00000000..cc66d4a9 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/archive.rb @@ -0,0 +1,8 @@ +module Travis + module Addons + module Archive + require 'travis/addons/archive/event_handler' + require 'travis/addons/archive/task' + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/archive/event_handler.rb b/vendor/travis-core/lib/travis/addons/archive/event_handler.rb new file mode 100644 index 00000000..725c251d --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/archive/event_handler.rb @@ -0,0 +1,36 @@ +require 'travis/addons/archive/task' +require 'travis/event/handler' +require 'travis/features' + +module Travis + module Addons + module Archive + class EventHandler < Event::Handler + EVENTS = /log:aggregated/ + + def handle? + Travis::Features.feature_active?(:log_archiving) + end + + def handle + Travis::Addons::Archive::Task.run(:archive, payload) + end + + def payload + @payload ||= { type: type, id: object.id, job_id: object.job_id } + end + + def type + @type ||= event.split(':').first + end + + class Instrument < Notification::Instrument::EventHandler + def notify_completed + publish(payload: handler.payload) + end + end + Instrument.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/archive/task.rb b/vendor/travis-core/lib/travis/addons/archive/task.rb new file mode 100644 index 00000000..37b974ee --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/archive/task.rb @@ -0,0 +1,24 @@ +require 'travis/task' + +module Travis + module Addons + module Archive + class Task < Travis::Task + def process + Travis.run_service(:"archive_#{payload[:type]}", id: payload[:id], job_id: payload[:job_id]) + end + + class Instrument < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #<#{target.payload[:type].camelize} id=#{target.payload[:id]}>", + :object_type => target.payload[:type].camelize, + :object_id => target.payload[:id] + ) + end + end + Instrument.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/campfire.rb b/vendor/travis-core/lib/travis/addons/campfire.rb new file mode 100644 index 00000000..6aada6d5 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/campfire.rb @@ -0,0 +1,13 @@ +module Travis + module Addons + module Campfire + module Instruments + require 'travis/addons/campfire/instruments' + end + + require 'travis/addons/campfire/event_handler' + + class Task < ::Travis::Task; end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/campfire/event_handler.rb b/vendor/travis-core/lib/travis/addons/campfire/event_handler.rb new file mode 100644 index 00000000..0a312c81 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/campfire/event_handler.rb @@ -0,0 +1,33 @@ +require 'travis/addons/campfire/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module Campfire + + # Publishes a build notification to campfire rooms as defined in the + # configuration (`.travis.yml`). + # + # Campfire credentials are encrypted using the repository's ssl key. + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = /build:finished/ + + def handle? + !pull_request? && targets.present? && config.send_on_finished_for?(:campfire) + end + + def handle + Travis::Addons::Campfire::Task.run(:campfire, payload, targets: targets) + end + + def targets + @targets ||= config.notification_values(:campfire, :rooms) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/campfire/instruments.rb b/vendor/travis-core/lib/travis/addons/campfire/instruments.rb new file mode 100644 index 00000000..c3b55de9 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/campfire/instruments.rb @@ -0,0 +1,30 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Campfire + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:targets => handler.targets) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request'][:id], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :targets => task.targets, + :message => task.message + ) + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/email.rb b/vendor/travis-core/lib/travis/addons/email.rb new file mode 100644 index 00000000..743d3f3f --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/email.rb @@ -0,0 +1,10 @@ +module Travis + module Addons + module Email + + require 'travis/addons/email/instruments' + require 'travis/addons/email/event_handler' + class Task < ::Travis::Task; end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/email/event_handler.rb b/vendor/travis-core/lib/travis/addons/email/event_handler.rb new file mode 100644 index 00000000..afdd4170 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/email/event_handler.rb @@ -0,0 +1,56 @@ +require 'travis/addons/email/instruments' +require 'travis/event/handler' +require 'travis/model/broadcast' + +module Travis + module Addons + module Email + + # Sends out build notification emails using ActionMailer. + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = ['build:finished', 'build:canceled'] + + def handle? + !pull_request? && config.enabled?(:email) && config.send_on_finished_for?(:email) && recipients.present? + end + + def handle + Travis::Addons::Email::Task.run(:email, payload, recipients: recipients, broadcasts: broadcasts) + end + + def recipients + @recipients ||= begin + recipients = config.notification_values(:email, :recipients) + recipients = config.notifications[:recipients] if recipients.blank? # TODO deprecate recipients + recipients = default_recipients if recipients.blank? + Array(recipients).join(',').split(',').map(&:strip).select(&:present?).uniq + end + end + + private + + def pull_request? + build['pull_request'] + end + + def broadcasts + Broadcast.by_repo(object.repository).map do |broadcast| + { message: broadcast.message } + end + end + + def default_recipients + recipients = object.repository.users.map {|u| u.emails.map(&:email)}.flatten + recipients.keep_if do |r| + r == object.commit.author_email or + r == object.commit.committer_email + end + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/email/instruments.rb b/vendor/travis-core/lib/travis/addons/email/instruments.rb new file mode 100644 index 00000000..4cc5c3bd --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/email/instruments.rb @@ -0,0 +1,30 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Email + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:recipients => handler.recipients) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request_id'], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :email => task.type, + :recipients => task.recipients + ) + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/flowdock.rb b/vendor/travis-core/lib/travis/addons/flowdock.rb new file mode 100644 index 00000000..bbd62f1d --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/flowdock.rb @@ -0,0 +1,10 @@ +module Travis + module Addons + module Flowdock + require 'travis/addons/flowdock/instruments' + require 'travis/addons/flowdock/event_handler' + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/flowdock/event_handler.rb b/vendor/travis-core/lib/travis/addons/flowdock/event_handler.rb new file mode 100644 index 00000000..08167d63 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/flowdock/event_handler.rb @@ -0,0 +1,39 @@ +require 'travis/addons/flowdock/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module Flowdock + + # Publishes a build notification to Flowdock rooms as defined in the + # configuration (`.travis.yml`). + # + # Flowdock credentials are encrypted using the repository's ssl key. + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = /build:finished/ + + def initialize(*) + super + @payload = Api.data(object, for: 'event', version: 'v0', params: data) + end + + def handle? + !pull_request? && targets.present? && config.send_on_finished_for?(:flowdock) + end + + def handle + Travis::Addons::Flowdock::Task.run(:flowdock, payload, targets: targets) + end + + def targets + @targets ||= config.notification_values(:flowdock, :rooms) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/flowdock/instruments.rb b/vendor/travis-core/lib/travis/addons/flowdock/instruments.rb new file mode 100644 index 00000000..421132ed --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/flowdock/instruments.rb @@ -0,0 +1,31 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Flowdock + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:targets => handler.targets) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request'][:id], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :targets => task.targets, + :message => task.message + ) + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/github_status.rb b/vendor/travis-core/lib/travis/addons/github_status.rb new file mode 100644 index 00000000..d3886d74 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/github_status.rb @@ -0,0 +1,10 @@ +module Travis + module Addons + module GithubStatus + require 'travis/addons/github_status/instruments' + require 'travis/addons/github_status/event_handler' + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/github_status/event_handler.rb b/vendor/travis-core/lib/travis/addons/github_status/event_handler.rb new file mode 100644 index 00000000..2aae7fa5 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/github_status/event_handler.rb @@ -0,0 +1,74 @@ +require 'travis/addons/github_status/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module GithubStatus + + # Adds a comment with a build notification to the pull-request the request + # belongs to. + class EventHandler < Event::Handler + API_VERSION = 'v2' + EVENTS = /build:(created|started|finished|canceled)/ + + def handle? + return token.present? unless multi_token? + + unless tokens.any? + error "No GitHub OAuth tokens found for #{object.repository.slug}" + end + + tokens.any? + end + + def handle + if multi_token? + Travis::Addons::GithubStatus::Task.run(:github_status, payload, tokens: tokens) + else + Travis::Addons::GithubStatus::Task.run(:github_status, payload, token: token) + end + end + + private + + def token + admin.try(:github_oauth_token) + end + + def tokens + @tokens ||= users.map { |user| { user.login => user.github_oauth_token } }.inject({}, :merge) + end + + def users + @users ||= [ + build_committer, + admin, + users_with_push_access, + ].flatten.compact + end + + def build_committer + user = User.with_email(object.commit.committer_email) + user if user && user.permission?(repository_id: object.repository.id, push: true) + end + + def admin + @admin ||= Travis.run_service(:find_admin, repository: object.repository) + rescue Travis::AdminMissing + nil + end + + def users_with_push_access + object.repository.users_with_permission(:push) + end + + def multi_token? + !Travis::Features.feature_deactivated?(:github_status_multi_tokens) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/github_status/instruments.rb b/vendor/travis-core/lib/travis/addons/github_status/instruments.rb new file mode 100644 index 00000000..525742fa --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/github_status/instruments.rb @@ -0,0 +1,30 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module GithubStatus + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request_id'], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :url => task.url.to_s + ) + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/hipchat.rb b/vendor/travis-core/lib/travis/addons/hipchat.rb new file mode 100644 index 00000000..b8664ff8 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/hipchat.rb @@ -0,0 +1,10 @@ +module Travis + module Addons + module Hipchat + require 'travis/addons/hipchat/instruments' + require 'travis/addons/hipchat/event_handler' + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/hipchat/event_handler.rb b/vendor/travis-core/lib/travis/addons/hipchat/event_handler.rb new file mode 100644 index 00000000..73a8657e --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/hipchat/event_handler.rb @@ -0,0 +1,40 @@ +require 'travis/addons/hipchat/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module Hipchat + + # Publishes a build notification to hipchat rooms as defined in the + # configuration (`.travis.yml`). + # + # Hipchat credentials are encrypted using the repository's ssl key. + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = /build:finished/ + + def handle? + enabled? && targets.present? && config.send_on_finished_for?(:hipchat) + end + + def handle + Travis::Addons::Hipchat::Task.run(:hipchat, payload, targets: targets) + end + + def enabled? + enabled = config.notification_values(:hipchat, :on_pull_requests) + enabled = true if enabled.nil? + pull_request? ? enabled : true + end + + def targets + @targets ||= config.notification_values(:hipchat, :rooms) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/hipchat/instruments.rb b/vendor/travis-core/lib/travis/addons/hipchat/instruments.rb new file mode 100644 index 00000000..b1fa6472 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/hipchat/instruments.rb @@ -0,0 +1,31 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Hipchat + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:targets => handler.targets) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request'][:id], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :targets => task.targets, + :message => task.message + ) + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/irc.rb b/vendor/travis-core/lib/travis/addons/irc.rb new file mode 100644 index 00000000..f5fc6133 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/irc.rb @@ -0,0 +1,10 @@ +module Travis + module Addons + module Irc + require 'travis/addons/irc/instruments' + require 'travis/addons/irc/event_handler' + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/irc/event_handler.rb b/vendor/travis-core/lib/travis/addons/irc/event_handler.rb new file mode 100644 index 00000000..dc50ae9b --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/irc/event_handler.rb @@ -0,0 +1,32 @@ +require 'travis/addons/irc/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module Irc + + # Publishes a build notification to IRC channels as defined in the + # configuration (`.travis.yml`). + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = 'build:finished' + + def handle? + !pull_request? && channels.present? && config.send_on_finished_for?(:irc) + end + + def handle + Travis::Addons::Irc::Task.run(:irc, payload, channels: channels) + end + + def channels + @channels ||= config.notification_values(:irc, :channels) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/irc/instruments.rb b/vendor/travis-core/lib/travis/addons/irc/instruments.rb new file mode 100644 index 00000000..ee013770 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/irc/instruments.rb @@ -0,0 +1,31 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Irc + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:channels => handler.channels) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request_id'], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :channels => task.channels, + :messages => task.messages + ) + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/pusher.rb b/vendor/travis-core/lib/travis/addons/pusher.rb new file mode 100644 index 00000000..90564fc3 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/pusher.rb @@ -0,0 +1,11 @@ +module Travis + module Addons + module Pusher + require 'travis/addons/pusher/instruments' + require 'travis/addons/pusher/event_handler' + + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/pusher/event_handler.rb b/vendor/travis-core/lib/travis/addons/pusher/event_handler.rb new file mode 100644 index 00000000..27c3d620 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/pusher/event_handler.rb @@ -0,0 +1,57 @@ +require 'travis/addons/pusher/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module Pusher + + # Notifies registered clients about various state changes through Pusher. + class EventHandler < Event::Handler + EVENTS = [ + /^build:(created|received|started|finished|canceled)/, + /^job:test:(created|received|started|log|finished|canceled)/ + ] + + attr_reader :channels, :pusher_payload + + def initialize(*) + super + @pusher_payload = Api.data(object, :for => 'pusher', :type => type, :params => data) if handle? + end + + def handle? + true + end + + def handle + Travis::Addons::Pusher::Task.run(queue, pusher_payload, :event => event) + end + + private + + def type + event.sub('test:', '').sub(':', '/') + end + + def queue + if Travis::Features.enabled_for_all?(:"pusher-live") || + Travis::Features.repository_active?(:"pusher-live", repository_id) + :"pusher-live" + else + :pusher + end + end + + def repository_id + if payload && payload['repository'] && payload['repository']['id'] + payload['repository']['id'] + elsif object && object.repository && object.repository.id + object.repository.id + end + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/pusher/instruments.rb b/vendor/travis-core/lib/travis/addons/pusher/instruments.rb new file mode 100644 index 00000000..97d5f5ff --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/pusher/instruments.rb @@ -0,0 +1,42 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Pusher + module Instruments + def self.publish?(event) + event.to_s != 'job:test:log' + end + + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish if Instruments.publish?(handler.event) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #<#{type.camelize} id=#{id}> (event: #{task.event}, channels: #{task.channels.join(', ')})", + :object_type => type.camelize, + :object_id => id, + :event => task.event, + :client_event => task.client_event, + :channels => task.channels + ) if Instruments.publish?(task.event) + end + + def type + @type ||= task.event.split(':').first + end + + def id + payload.key?(type.to_sym) ? payload[type.to_sym][:id] : payload[:id] + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/pushover.rb b/vendor/travis-core/lib/travis/addons/pushover.rb new file mode 100644 index 00000000..10cbd194 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/pushover.rb @@ -0,0 +1,13 @@ +module Travis + module Addons + module Pushover + module Instruments + require 'travis/addons/pushover/instruments' + end + + require 'travis/addons/pushover/event_handler' + + class Task < ::Travis::Task; end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/pushover/event_handler.rb b/vendor/travis-core/lib/travis/addons/pushover/event_handler.rb new file mode 100644 index 00000000..123e9495 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/pushover/event_handler.rb @@ -0,0 +1,37 @@ +require 'travis/addons/pushover/instruments' +require 'travis/event/handler' + +module Travis + module Addons + module Pushover + + # Publishes a build notification to pushover users as defined in the + # configuration (`.travis.yml`). + # + # Credentials are encrypted using the repository's ssl key. + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = /build:finished/ + + def handle? + !pull_request? && users.present? && api_key.present? && config.send_on_finished_for?(:pushover) + end + + def handle + Travis::Addons::Pushover::Task.run(:pushover, payload, users: users, api_key: api_key) + end + + def users + @users ||= config.notification_values(:pushover, :users) + end + + def api_key + @api_key ||= config.notifications[:pushover][:api_key] + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/pushover/instruments.rb b/vendor/travis-core/lib/travis/addons/pushover/instruments.rb new file mode 100644 index 00000000..6ed31392 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/pushover/instruments.rb @@ -0,0 +1,30 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Pushover + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:users => handler.users, :api_key => handler.api_key) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + :object_type => 'Build', + :object_id => payload[:build][:id], + :users => task.users, + :message => task.message, + :api_key => task.api_key + ) + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/slack.rb b/vendor/travis-core/lib/travis/addons/slack.rb new file mode 100644 index 00000000..1d682235 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/slack.rb @@ -0,0 +1,11 @@ +module Travis + module Addons + module Slack + require 'travis/addons/slack/instruments' + require 'travis/addons/slack/event_handler' + + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/slack/event_handler.rb b/vendor/travis-core/lib/travis/addons/slack/event_handler.rb new file mode 100644 index 00000000..9cfa2c80 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/slack/event_handler.rb @@ -0,0 +1,30 @@ +module Travis + module Addons + module Slack + + # Publishes a build notification to Slack rooms as defined in the + # configuration (`.travis.yml`). + # + # Slack credentials are encrypted using the repository's ssl key. + class EventHandler < Event::Handler + API_VERSION = 'v2' + + EVENTS = /build:finished/ + + def handle? + targets.present? && config.send_on_finished_for?(:slack) + end + + def handle + Travis::Addons::Slack::Task.run(:slack, payload, targets: targets) + end + + def targets + @targets ||= config.notification_values(:slack, :rooms) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/slack/instruments.rb b/vendor/travis-core/lib/travis/addons/slack/instruments.rb new file mode 100644 index 00000000..ca3de728 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/slack/instruments.rb @@ -0,0 +1,25 @@ +module Travis + module Addons + module Slack + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:targets => handler.targets) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + :object_type => 'Build', + :object_id => payload[:build][:id], + :targets => task.targets + ) + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/sqwiggle.rb b/vendor/travis-core/lib/travis/addons/sqwiggle.rb new file mode 100644 index 00000000..17829b38 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/sqwiggle.rb @@ -0,0 +1,11 @@ +module Travis + module Addons + module Sqwiggle + require 'travis/addons/sqwiggle/instruments' + require 'travis/addons/sqwiggle/event_handler' + + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/sqwiggle/event_handler.rb b/vendor/travis-core/lib/travis/addons/sqwiggle/event_handler.rb new file mode 100644 index 00000000..a12ba383 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/sqwiggle/event_handler.rb @@ -0,0 +1,30 @@ +module Travis + module Addons + module Sqwiggle + + # Publishes a build notification to sqwiggle rooms as defined in the + # configuration (`.travis.yml`). + # + # sqwiggle credentials are encrypted using the repository's ssl key. + class EventHandler < Event::Handler + + EVENTS = /build:finished/ + + def handle? + !pull_request? && targets.present? && config.send_on_finished_for?(:sqwiggle) + end + + def handle + Travis::Addons::Sqwiggle::Task.run(:sqwiggle, payload, targets: targets) + end + + def targets + @targets ||= config.notification_values(:sqwiggle, :rooms) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/sqwiggle/instruments.rb b/vendor/travis-core/lib/travis/addons/sqwiggle/instruments.rb new file mode 100644 index 00000000..6daa5d79 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/sqwiggle/instruments.rb @@ -0,0 +1,29 @@ +module Travis + module Addons + module Sqwiggle + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:targets => handler.targets) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository][:slug], + # :request_id => payload['request'][:id], # TODO + :object_type => 'Build', + :object_id => payload[:build][:id], + :targets => task.targets, + :message => task.message + ) + end + end + end + end + end +end + + diff --git a/vendor/travis-core/lib/travis/addons/states_cache.rb b/vendor/travis-core/lib/travis/addons/states_cache.rb new file mode 100644 index 00000000..c33a1ea2 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/states_cache.rb @@ -0,0 +1,7 @@ +module Travis + module Addons + module StatesCache + require 'travis/addons/states_cache/event_handler' + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/states_cache/event_handler.rb b/vendor/travis-core/lib/travis/addons/states_cache/event_handler.rb new file mode 100644 index 00000000..a786287c --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/states_cache/event_handler.rb @@ -0,0 +1,47 @@ +require 'travis/event/handler' + +module Travis + module Addons + module StatesCache + class EventHandler < Event::Handler + EVENTS = /build:finished/ + + def handle? + states_cache_enabled = Travis::Features.feature_active?(:states_cache) + result = !pull_request? && states_cache_enabled + Travis.logger.info("[states-cache] Checking if event handler should be run for " + + "repo_id=#{repository_id} branch=#{branch} build_id=#{build['id']}, result: #{result}, " + + "pull_request: #{pull_request?} states_cache_enabled: #{states_cache_enabled}") + result + end + + def handle + Travis.logger.info("[states-cache] Running event handler for repo_id=#{repository_id} build_id=#{build['id']} branch=#{branch}") + cache.write(repository_id, branch, data) + rescue Exception => e + Travis.logger.error("[states-cache] An error occurred while trying to handle states cache update: #{e.message}\n#{e.backtrace}") + raise + end + + def cache + Travis.states_cache + end + + def repository_id + build['repository_id'] + end + + def branch + commit['branch'] + end + + def data + { + 'id' => build['id'], + 'state' => build['state'] + } + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/util.rb b/vendor/travis-core/lib/travis/addons/util.rb new file mode 100644 index 00000000..3a0e1075 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/util.rb @@ -0,0 +1,7 @@ +module Travis + module Addons + module Util + require 'travis/addons/util/template' + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/webhook.rb b/vendor/travis-core/lib/travis/addons/webhook.rb new file mode 100644 index 00000000..3c03922c --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/webhook.rb @@ -0,0 +1,11 @@ +module Travis + module Addons + module Webhook + require 'travis/addons/webhook/instruments' + require 'travis/addons/webhook/event_handler' + + class Task < ::Travis::Task; end + end + end +end + diff --git a/vendor/travis-core/lib/travis/addons/webhook/event_handler.rb b/vendor/travis-core/lib/travis/addons/webhook/event_handler.rb new file mode 100644 index 00000000..129757d0 --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/webhook/event_handler.rb @@ -0,0 +1,39 @@ +require 'travis/addons/webhook/instruments' +require 'travis/event/handler' + +# TODO include_logs? has been removed. gotta be deprecated! +# +module Travis + module Addons + module Webhook + + # Sends build notifications to webhooks as defined in the configuration + # (`.travis.yml`). + class EventHandler < Event::Handler + EVENTS = /build:(started|finished)/ + + def initialize(*) + super + end + + def handle? + targets.present? && config.send_on?(:webhooks, event.split(':').last) + end + + def handle + Travis::Addons::Webhook::Task.run(:webhook, webhook_payload, targets: targets, token: request['token']) + end + + def webhook_payload + Api.data(object, :for => 'webhook', :type => 'build/finished', :version => 'v1') + end + + def targets + @targets ||= config.notification_values(:webhooks, :urls) + end + + Instruments::EventHandler.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/addons/webhook/instruments.rb b/vendor/travis-core/lib/travis/addons/webhook/instruments.rb new file mode 100644 index 00000000..9552f04d --- /dev/null +++ b/vendor/travis-core/lib/travis/addons/webhook/instruments.rb @@ -0,0 +1,29 @@ +require 'travis/notification/instrument/event_handler' +require 'travis/notification/instrument/task' + +module Travis + module Addons + module Webhook + module Instruments + class EventHandler < Notification::Instrument::EventHandler + def notify_completed + publish(:targets => handler.targets) + end + end + + class Task < Notification::Instrument::Task + def run_completed + publish( + :msg => "for #", + :repository => payload[:repository].values_at(:owner_name, :name).join('/'), + :object_type => 'Build', + :object_id => payload[:id], + :targets => task.targets + ) + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/advisory_locks.rb b/vendor/travis-core/lib/travis/advisory_locks.rb new file mode 100644 index 00000000..4edae945 --- /dev/null +++ b/vendor/travis-core/lib/travis/advisory_locks.rb @@ -0,0 +1,59 @@ +require 'zlib' + +module Travis + # http://hashrocket.com/blog/posts/advisory-locks-in-postgres + # https://github.com/mceachen/with_advisory_lock + # 13.3.4. Advisory Locks : http://www.postgresql.org/docs/9.3/static/explicit-locking.html + # http://www.postgresql.org/docs/9.3/static/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS + class AdvisoryLocks + attr_reader :lock_name, :transactional + + def initialize(lock_name, transactional = false) + if lock_name.blank? + raise StandardError, "lock name cannot be blank" + end + @lock_name = lock_name + end + + # must be used within a transaction + def self.exclusive(lock_name, timeout = 30, transactional = false) + al = self.new(lock_name, transactional) + al.exclusive(timeout) { yield } + end + + # must be used within a transaction + def exclusive(timeout = 30) + give_up_at = Time.now + timeout if timeout + while timeout.nil? || Time.now < give_up_at do + if obtained_lock? + return yield + else + # Randomizing sleep time may help reduce contention. + sleep(rand(0.1..0.2)) + end + end + ensure + release_lock unless transactional + end + + private + + def obtained_lock? + xact = transactional ? "_xact" : nil + result = connection.select_value("select pg_try_advisory#{xact}_lock(#{lock_code});") + result == 't' || result == 'true' + end + + def release_lock + connection.execute("select pg_advisory_unlock(#{lock_code});") + end + + def connection + ActiveRecord::Base.connection + end + + def lock_code + Zlib.crc32(lock_name) + end + end +end \ No newline at end of file diff --git a/vendor/travis-core/lib/travis/api.rb b/vendor/travis-core/lib/travis/api.rb new file mode 100644 index 00000000..b3991d7b --- /dev/null +++ b/vendor/travis-core/lib/travis/api.rb @@ -0,0 +1,67 @@ +module Travis + module Api + require 'travis/api/formats' + require 'travis/api/v0' + require 'travis/api/v1' + require 'travis/api/v2' + + DEFAULT_VERSION = 'v2' + + class << self + def data(resource, options = {}) + new(resource, options).data + end + + def builder(resource, options = {}) + target = (options[:for] || 'http').to_s.camelize + version = (options[:version] || default_version(options)).to_s.camelize + type = (options[:type] || type_for(resource)).to_s.camelize.split('::') + ([version, target] + type).inject(Travis::Api) do |const, name| + begin + if const && const.const_defined?(name.to_s.camelize, false) + const.const_get(name, false) + else + nil + end + rescue NameError + nil + end + end + end + + def new(resource, options = {}) + builder = builder(resource, options) || raise(ArgumentError, "cannot serialize #{resource.inspect}, options: #{options.inspect}") + builder.new(resource, options[:params] || {}) + end + + private + + def type_for(resource) + if arel_relation?(resource) + type = resource.klass.name.pluralize + else + type = resource.class + type = type.base_class if active_record?(type) + type = type.name + end + type.split('::').last + end + + def arel_relation?(object) + object.respond_to?(:klass) + end + + def active_record?(object) + object.respond_to?(:base_class) + end + + def default_version(options) + if options[:for].to_s.downcase == "pusher" + "v0" + else + DEFAULT_VERSION + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/README.markdown b/vendor/travis-core/lib/travis/api/README.markdown new file mode 100644 index 00000000..0ca565b1 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/README.markdown @@ -0,0 +1,10 @@ +This directory contains serializers for events and models. + +- `v0/event`: Payloads used by [`Travis::Event::Handler`](../event/handler.rb). These are the payloads that the [addons](../addons) will get. +- `v0/pusher`: Payloads used to send events to the web UI using Pusher. +- `v0/worker`: Payloads sent to [travis-worker](https://github.com/travis-ci/travis-worker). + +- `v1/http`: Payloads for the v1 [API](https://github.com/travis-ci/travis-api). +- `v1/webhook`: Payloads for the webhook notifications. + +- `v2/http`: Payloads for the v2 [API](https://github.com/travis-ci/travis-api). \ No newline at end of file diff --git a/vendor/travis-core/lib/travis/api/formats.rb b/vendor/travis-core/lib/travis/api/formats.rb new file mode 100644 index 00000000..b218bda5 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/formats.rb @@ -0,0 +1,9 @@ +module Travis + module Api + module Formats + def format_date(date) + date && date.strftime('%Y-%m-%dT%H:%M:%SZ') + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0.rb b/vendor/travis-core/lib/travis/api/v0.rb new file mode 100644 index 00000000..d21ff344 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0.rb @@ -0,0 +1,11 @@ +module Travis + module Api + # V0 is an internal api that we can change at any time + module V0 + require 'travis/api/v0/event' + require 'travis/api/v0/notification' + require 'travis/api/v0/pusher' + require 'travis/api/v0/worker' + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/event.rb b/vendor/travis-core/lib/travis/api/v0/event.rb new file mode 100644 index 00000000..efd03a8d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/event.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Event + require 'travis/api/v0/event/build' + require 'travis/api/v0/event/job' + end + end + end +end + + diff --git a/vendor/travis-core/lib/travis/api/v0/event/build.rb b/vendor/travis-core/lib/travis/api/v0/event/build.rb new file mode 100644 index 00000000..797d399a --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/event/build.rb @@ -0,0 +1,95 @@ +module Travis + module Api + module V0 + module Event + class Build + include Formats + + attr_reader :build, :repository, :request, :commit, :options + + def initialize(build, options = {}) + @build = build + @repository = build.repository + @request = build.request + @commit = build.commit + # @options = options + end + + def data(extra = {}) + { + 'repository' => repository_data, + 'request' => request_data, + 'commit' => commit_data, + 'build' => build_data, + 'jobs' => build.matrix.map { |job| job_data(job) } + } + end + + private + + def build_data + { + 'id' => build.id, + 'repository_id' => build.repository_id, + 'commit_id' => build.commit_id, + 'number' => build.number, + 'pull_request' => build.pull_request?, + 'pull_request_number' => build.pull_request_number, + 'config' => build.config.try(:except, :source_key), + 'state' => build.state.to_s, + 'previous_state' => build.previous_state.to_s, + 'started_at' => format_date(build.started_at), + 'finished_at' => format_date(build.finished_at), + 'duration' => build.duration, + 'job_ids' => build.matrix_ids + } + end + + def repository_data + { + 'id' => repository.id, + 'key' => repository.key.try(:public_key), + 'slug' => repository.slug, + 'name' => repository.name, + 'owner_email' => repository.owner_email, + 'owner_avatar_url' => repository.owner.try(:avatar_url) + } + end + + def request_data + { + 'token' => request.token, + 'head_commit' => (request.head_commit || '') + } + end + + def commit_data + { + 'id' => commit.id, + 'sha' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'compare_url' => commit.compare_url, + } + end + + def job_data(job) + { + 'id' => job.id, + 'number' => job.number, + 'state' => job.state.to_s, + 'tags' => job.tags + } + end + end + end + end + end +end + + diff --git a/vendor/travis-core/lib/travis/api/v0/event/job.rb b/vendor/travis-core/lib/travis/api/v0/event/job.rb new file mode 100644 index 00000000..ddbc58e8 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/event/job.rb @@ -0,0 +1,38 @@ +module Travis + module Api + module V0 + module Event + class Job + include Formats + + attr_reader :job + + def initialize(job, options = {}) + @job = job + # @options = options + end + + def data(extra = {}) + { + 'job' => job_data, + } + end + + private + + def job_data + { + 'queue' => job.queue, + 'created_at' => job.created_at, + 'started_at' => job.started_at, + 'finished_at' => job.finished_at, + } + end + end + end + end + end +end + + + diff --git a/vendor/travis-core/lib/travis/api/v0/notification.rb b/vendor/travis-core/lib/travis/api/v0/notification.rb new file mode 100644 index 00000000..45d37770 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/notification.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Notification + require 'travis/api/v0/notification/build' + require 'travis/api/v0/notification/repository' + require 'travis/api/v0/notification/user' + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v0/notification/build.rb b/vendor/travis-core/lib/travis/api/v0/notification/build.rb new file mode 100644 index 00000000..27b00504 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/notification/build.rb @@ -0,0 +1,28 @@ +module Travis + module Api + module V0 + module Notification + class Build + attr_reader :build + + def initialize(build, options = {}) + @build = build + end + + def data + { + 'build' => build_data + } + end + + def build_data + { + 'id' => build.id + } + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v0/notification/repository.rb b/vendor/travis-core/lib/travis/api/v0/notification/repository.rb new file mode 100644 index 00000000..6b9861b8 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/notification/repository.rb @@ -0,0 +1,28 @@ +module Travis + module Api + module V0 + module Notification + class Repository + attr_reader :repository + + def initialize(repository, options = {}) + @repository = repository + end + + def data + { + 'repository' => repository_data + } + end + + def repository_data + { + 'id' => repository.id, + 'slug' => repository.slug + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/notification/user.rb b/vendor/travis-core/lib/travis/api/v0/notification/user.rb new file mode 100644 index 00000000..95ba9192 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/notification/user.rb @@ -0,0 +1,28 @@ +module Travis + module Api + module V0 + module Notification + class User + attr_reader :user + + def initialize(user, options = {}) + @user = user + end + + def data + { + 'user' => user_data + } + end + + def user_data + { + 'id' => user.id, + 'login' => user.login + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher.rb b/vendor/travis-core/lib/travis/api/v0/pusher.rb new file mode 100644 index 00000000..852c3045 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher.rb @@ -0,0 +1,11 @@ +module Travis + module Api + module V0 + module Pusher + require 'travis/api/v0/pusher/annotation' + require 'travis/api/v0/pusher/build' + require 'travis/api/v0/pusher/job' + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/annotation.rb b/vendor/travis-core/lib/travis/api/v0/pusher/annotation.rb new file mode 100644 index 00000000..f29f3d42 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/annotation.rb @@ -0,0 +1,33 @@ +module Travis + module Api + module V0 + module Pusher + class Annotation + require 'travis/api/v0/pusher/annotation/created' + require 'travis/api/v0/pusher/annotation/updated' + + include Formats + + attr_reader :annotation + + def initialize(annotation, options = {}) + @annotation = annotation + end + + def data + { + "annotation" => { + "id" => annotation.id, + "job_id" => annotation.job_id, + "description" => annotation.description, + "url" => annotation.url, + "status" => annotation.status, + "provider_name" => annotation.annotation_provider.name, + } + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/annotation/created.rb b/vendor/travis-core/lib/travis/api/v0/pusher/annotation/created.rb new file mode 100644 index 00000000..fb476fd1 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/annotation/created.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Annotation + class Created < Annotation + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/annotation/updated.rb b/vendor/travis-core/lib/travis/api/v0/pusher/annotation/updated.rb new file mode 100644 index 00000000..99865c40 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/annotation/updated.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Annotation + class Updated < Annotation + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build.rb new file mode 100644 index 00000000..3762aa38 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build.rb @@ -0,0 +1,111 @@ +module Travis + module Api + module V0 + module Pusher + class Build + require 'travis/api/v0/pusher/build/canceled' + require 'travis/api/v0/pusher/build/created' + require 'travis/api/v0/pusher/build/received' + require 'travis/api/v0/pusher/build/started' + require 'travis/api/v0/pusher/build/finished' + + include Formats + + attr_reader :build, :options + + def initialize(build, options = {}) + @build = build + @options = options + end + + def data + { + 'build' => build_data(build), + 'commit' => commit_data(build.commit), + 'repository' => repository_data(build.repository) + } + end + + private + + def build_data(build) + commit = build.commit + { + 'id' => build.id, + 'repository_id' => build.repository_id, + 'commit_id' => build.commit_id, + 'number' => build.number, + 'pull_request' => build.pull_request?, + 'pull_request_title' => build.pull_request_title, + 'pull_request_number' => build.pull_request_number, + 'state' => build.state.to_s, + 'started_at' => format_date(build.started_at), + 'finished_at' => format_date(build.finished_at), + 'duration' => build.duration, + 'job_ids' => build.matrix_ids, + 'event_type' => build.event_type, + + # this is a legacy thing, we should think about removing it + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'compare_url' => commit.compare_url, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email + } + end + + def commit_data(commit) + { + 'id' => commit.id, + 'sha' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'compare_url' => commit.compare_url, + } + end + + def repository_data(repository) + { + 'id' => repository.id, + 'slug' => repository.slug, + 'description' => repository.description, + 'private' => repository.private, + 'last_build_id' => repository.last_build_id, + 'last_build_number' => repository.last_build_number, + 'last_build_state' => repository.last_build_state.to_s, + 'last_build_duration' => repository.last_build_duration, + 'last_build_language' => nil, + 'last_build_started_at' => format_date(repository.last_build_started_at), + 'last_build_finished_at' => format_date(repository.last_build_finished_at), + 'github_language' => repository.github_language, + 'default_branch' => { + 'name' => repository.default_branch, + 'last_build_id' => last_build_on_default_branch_id(repository) + }, + 'active' => repository.active, + 'current_build_id' => repository.current_build_id + } + end + + def last_build_on_default_branch_id(repository) + default_branch = Branch.where(repository_id: repository.id, name: repository.default_branch).first + + if default_branch + default_branch.last_build_id + end + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/canceled.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/canceled.rb new file mode 100644 index 00000000..19ae8256 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/canceled.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Build + class Canceled < Build + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/created.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/created.rb new file mode 100644 index 00000000..2a0d008e --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/created.rb @@ -0,0 +1,15 @@ +require 'travis/api/v1' + +module Travis + module Api + module V0 + module Pusher + class Build + class Created < Build + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/finished.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/finished.rb new file mode 100644 index 00000000..299df81d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/finished.rb @@ -0,0 +1,14 @@ +module Travis + module Api + module V0 + module Pusher + class Build + class Finished < Build + end + end + end + end + end +end + + diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/received.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/received.rb new file mode 100644 index 00000000..005f2ab2 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/received.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Build + class Received < Build + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/received/job.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/received/job.rb new file mode 100644 index 00000000..6786f598 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/received/job.rb @@ -0,0 +1,47 @@ +module Travis + module Api + module V0 + module Pusher + class Build + class Received < Build + class Job + include Formats, V1::Helpers::Legacy + + attr_reader :job, :commit + + def initialize(job) + @job = job + @commit = job.commit + end + + def data + { + 'id' => job.id, + 'repository_id' => job.repository_id, + 'repository_private' => repository.private, + 'parent_id' => job.source_id, + 'number' => job.number, + 'state' => job.state.to_s, + 'result' => legacy_job_result(job), + 'config' => job.obfuscated_config, + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'compare_url' => commit.compare_url, + 'started_at' => format_date(job.started_at), + 'finished_at' => format_date(job.finished_at), + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'allow_failure' => job.allow_failure + } + end + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/started.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/started.rb new file mode 100644 index 00000000..1ad598b0 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/started.rb @@ -0,0 +1,13 @@ +module Travis + module Api + module V0 + module Pusher + class Build + class Started < Build + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/build/started/job.rb b/vendor/travis-core/lib/travis/api/v0/pusher/build/started/job.rb new file mode 100644 index 00000000..4a94660b --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/build/started/job.rb @@ -0,0 +1,47 @@ +module Travis + module Api + module V0 + module Pusher + class Build + class Started < Build + class Job + include Formats, V1::Helpers::Legacy + + attr_reader :job, :commit + + def initialize(job) + @job = job + @commit = job.commit + end + + def data + { + 'id' => job.id, + 'repository_id' => job.repository_id, + 'repository_private' => repository.private, + 'parent_id' => job.source_id, + 'number' => job.number, + 'state' => job.state.to_s, + 'result' => legacy_job_result(job), + 'config' => job.obfuscated_config, + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'compare_url' => commit.compare_url, + 'started_at' => format_date(job.started_at), + 'finished_at' => format_date(job.finished_at), + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'allow_failure' => job.allow_failure + } + end + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job.rb new file mode 100644 index 00000000..efd3cbc8 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job.rb @@ -0,0 +1,67 @@ +module Travis + module Api + module V0 + module Pusher + class Job + require 'travis/api/v0/pusher/job/canceled' + require 'travis/api/v0/pusher/job/created' + require 'travis/api/v0/pusher/job/log' + require 'travis/api/v0/pusher/job/received' + require 'travis/api/v0/pusher/job/started' + require 'travis/api/v0/pusher/job/finished' + + include Formats + + attr_reader :job, :options + + def initialize(job, options = {}) + @job = job + @options = options + end + + def data + job_data(job).merge( + 'commit' => commit_data(job.commit) + ) + end + + private + + def job_data(job) + { + 'id' => job.id, + 'repository_id' => job.repository_id, + 'repository_slug' => job.repository.slug, + 'repository_private' => job.repository.private, + 'build_id' => job.source_id, + 'commit_id' => job.commit_id, + 'log_id' => job.log_id, + 'number' => job.number, + 'state' => job.state.to_s, + 'started_at' => format_date(job.started_at), + 'finished_at' => format_date(job.finished_at), + 'queue' => job.queue, + 'allow_failure' => job.allow_failure, + 'annotation_ids' => job.annotation_ids + } + end + + def commit_data(commit) + { + 'id' => commit.id, + 'sha' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'compare_url' => commit.compare_url, + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job/canceled.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job/canceled.rb new file mode 100644 index 00000000..426d03f3 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job/canceled.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Job + class Canceled < Job + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job/created.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job/created.rb new file mode 100644 index 00000000..c2af620e --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job/created.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Job + class Created < Job + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job/finished.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job/finished.rb new file mode 100644 index 00000000..2214041c --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job/finished.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Job + class Finished < Job + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job/log.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job/log.rb new file mode 100644 index 00000000..54115ee6 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job/log.rb @@ -0,0 +1,31 @@ +module Travis + module Api + module V0 + module Pusher + class Job + class Log + attr_reader :job, :options + + def initialize(job, options = {}) + @job = job + @options = options + end + + def data + { + 'id' => job.id, + 'build_id' => job.source_id, + 'repository_id' => job.repository_id, + 'repository_private' => repository.private, + '_log' => options[:_log], + 'number' => options[:number], + 'final' => options[:final] + } + end + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job/received.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job/received.rb new file mode 100644 index 00000000..0921280d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job/received.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Job + class Received < Job + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/pusher/job/started.rb b/vendor/travis-core/lib/travis/api/v0/pusher/job/started.rb new file mode 100644 index 00000000..90ccc963 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/pusher/job/started.rb @@ -0,0 +1,12 @@ +module Travis + module Api + module V0 + module Pusher + class Job + class Started < Job + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/worker.rb b/vendor/travis-core/lib/travis/api/v0/worker.rb new file mode 100644 index 00000000..4826160e --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/worker.rb @@ -0,0 +1,9 @@ +module Travis + module Api + module V0 + module Worker + require 'travis/api/v0/worker/job' + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/worker/job.rb b/vendor/travis-core/lib/travis/api/v0/worker/job.rb new file mode 100644 index 00000000..796d0620 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/worker/job.rb @@ -0,0 +1,33 @@ +module Travis + module Api + module V0 + module Worker + class Job + require 'travis/api/v0/worker/job/test' + + attr_reader :job + + def initialize(job, options = {}) + @job = job + end + + def commit + job.commit + end + + def repository + job.repository + end + + def request + build.request + end + + def build + job.source + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v0/worker/job/test.rb b/vendor/travis-core/lib/travis/api/v0/worker/job/test.rb new file mode 100644 index 00000000..8462a9bc --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v0/worker/job/test.rb @@ -0,0 +1,118 @@ +module Travis + module Api + module V0 + module Worker + class Job + class Test < Job + include Formats + + def data + { + 'type' => 'test', + # TODO legacy. remove this once workers respond to a 'job' key + 'build' => job_data, + 'job' => job_data, + 'source' => build_data, + 'repository' => repository_data, + 'pull_request' => commit.pull_request? ? pull_request_data : false, + 'config' => job.decrypted_config, + 'queue' => job.queue, + 'uuid' => Travis.uuid, + 'ssh_key' => ssh_key, + 'env_vars' => env_vars, + 'timeouts' => timeouts + } + end + + def build_data + { + 'id' => build.id, + 'number' => build.number + } + end + + def job_data + data = { + 'id' => job.id, + 'number' => job.number, + 'commit' => commit.commit, + 'commit_range' => commit.range, + 'commit_message' => commit.message, + 'branch' => commit.branch, + 'ref' => commit.pull_request? ? commit.ref : nil, + 'state' => job.state.to_s, + 'secure_env_enabled' => job.secure_env_enabled? + } + data['tag'] = request.tag_name if include_tag_name? + data['pull_request'] = commit.pull_request? ? commit.pull_request_number : false + data + end + + def repository_data + { + 'id' => repository.id, + 'slug' => repository.slug, + 'github_id' => repository.github_id, + 'source_url' => repository.source_url, + 'api_url' => repository.api_url, + 'last_build_id' => repository.last_build_id, + 'last_build_number' => repository.last_build_number, + 'last_build_started_at' => format_date(repository.last_build_started_at), + 'last_build_finished_at' => format_date(repository.last_build_finished_at), + 'last_build_duration' => repository.last_build_duration, + 'last_build_state' => repository.last_build_state.to_s, + 'description' => repository.description, + 'default_branch' => repository.default_branch + } + end + + def pull_request_data + { + 'number' => commit.pull_request_number, + 'head_repo' => request.head_repo, + 'base_repo' => request.base_repo, + 'head_branch' => request.head_branch, + 'base_branch' => request.base_branch + } + end + + def ssh_key + nil + end + + def env_vars + vars = settings.env_vars + vars = vars.public unless job.secure_env_enabled? + + vars.map do |var| + { + 'name' => var.name, + 'value' => var.value.decrypt, + 'public' => var.public + } + end + end + + def timeouts + { 'hard_limit' => timeout(:hard_limit), 'log_silence' => timeout(:log_silence) } + end + + def timeout(type) + timeout = settings.send(:"timeout_#{type}") + timeout = timeout * 60 if timeout # worker handles timeouts in seconds + timeout + end + + def include_tag_name? + Travis.config.include_tag_name_in_worker_payload && request.tag_name.present? + end + + def settings + repository.settings + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1.rb b/vendor/travis-core/lib/travis/api/v1.rb new file mode 100644 index 00000000..bb29b348 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1.rb @@ -0,0 +1,10 @@ +module Travis + module Api + module V1 + require 'travis/api/v1/archive' + require 'travis/api/v1/http' + require 'travis/api/v1/helpers' + require 'travis/api/v1/webhook' + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/archive.rb b/vendor/travis-core/lib/travis/api/v1/archive.rb new file mode 100644 index 00000000..db343c81 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/archive.rb @@ -0,0 +1,9 @@ +module Travis + module Api + module V1 + module Archive + require 'travis/api/v1/archive/build' + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/archive/build.rb b/vendor/travis-core/lib/travis/api/v1/archive/build.rb new file mode 100644 index 00000000..9911cd0d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/archive/build.rb @@ -0,0 +1,50 @@ +module Travis + module Api + module V1 + module Archive + class Build + autoload :Job, 'travis/api/v1/archive/build/job' + + include Formats + + attr_reader :build, :commit, :repository + + def initialize(build, options = {}) + @build = build + @commit = build.commit + @repository = build.repository + end + + def data + { + 'id' => build.id, + 'number' => build.number, + 'config' => build.obfuscated_config.stringify_keys, + 'result' => 0, + 'started_at' => format_date(build.started_at), + 'finished_at' => format_date(build.finished_at), + 'duration' => build.duration, + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'matrix' => build.matrix.map { |job| Job.new(job).data }, + 'repository' => repository_data + } + end + + def repository_data + { + 'id' => repository.id, + 'slug' => repository.slug + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/archive/build/job.rb b/vendor/travis-core/lib/travis/api/v1/archive/build/job.rb new file mode 100644 index 00000000..110d971a --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/archive/build/job.rb @@ -0,0 +1,31 @@ +module Travis + module Api + module V1 + module Archive + class Build + class Job + include Formats + + attr_reader :job, :commit + + def initialize(job) + @job = job + @commit = job.commit + end + + def data + { + 'id' => job.id, + 'number' => job.number, + 'config' => job.obfuscated_config, + 'started_at' => format_date(job.started_at), + 'finished_at' => format_date(job.finished_at), + 'log' => job.log_content + } + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/helpers.rb b/vendor/travis-core/lib/travis/api/v1/helpers.rb new file mode 100644 index 00000000..c4905ed9 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/helpers.rb @@ -0,0 +1,9 @@ +module Travis + module Api + module V1 + module Helpers + require 'travis/api/v1/helpers/legacy' + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/helpers/legacy.rb b/vendor/travis-core/lib/travis/api/v1/helpers/legacy.rb new file mode 100644 index 00000000..2a8d3a8b --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/helpers/legacy.rb @@ -0,0 +1,34 @@ +module Travis + module Api + module V1 + module Helpers + module Legacy + RESULTS = { + passed: 0, + failed: 1 + } + + def legacy_repository_last_build_result(repository) + RESULTS[repository.last_build_state.try(:to_sym)] + end + + def legacy_build_state(build) + build.finished? ? 'finished' : build.state.to_s + end + + def legacy_build_result(build) + RESULTS[build.state.try(:to_sym)] + end + + def legacy_job_state(job) + job.finished? ? 'finished' : job.state.to_s + end + + def legacy_job_result(job) + RESULTS[job.state.try(:to_sym)] + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http.rb b/vendor/travis-core/lib/travis/api/v1/http.rb new file mode 100644 index 00000000..a7db9f95 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http.rb @@ -0,0 +1,17 @@ +module Travis + module Api + module V1 + module Http + require 'travis/api/v1/http/branches' + require 'travis/api/v1/http/build' + require 'travis/api/v1/http/builds' + require 'travis/api/v1/http/hooks' + require 'travis/api/v1/http/job' + require 'travis/api/v1/http/jobs' + require 'travis/api/v1/http/repositories' + require 'travis/api/v1/http/repository' + require 'travis/api/v1/http/user' + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/branches.rb b/vendor/travis-core/lib/travis/api/v1/http/branches.rb new file mode 100644 index 00000000..30e0c01d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/branches.rb @@ -0,0 +1,43 @@ +require 'travis/api/v1/helpers/legacy' + +module Travis + module Api + module V1 + module Http + class Branches + include Formats, Helpers::Legacy + + attr_reader :builds, :options + + def initialize(builds, options = {}) + builds = builds.last_finished_builds_by_branches if builds.is_a?(Repository) # TODO remove, bc + @builds = builds + end + + def cache_key + "branches-#{builds.map(&:id).join('-')}" + end + + def updated_at + builds.compact.map(&:finished_at).compact.sort.first + end + + def data + builds.compact.map do |build| + { + 'repository_id' => build.repository_id, + 'build_id' => build.id, + 'commit' => build.commit.commit, + 'branch' => build.commit.branch, + 'message' => build.commit.message, + 'result' => legacy_build_result(build), + 'finished_at' => format_date(build.finished_at), + 'started_at' => format_date(build.started_at) + } + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/build.rb b/vendor/travis-core/lib/travis/api/v1/http/build.rb new file mode 100644 index 00000000..e0431718 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/build.rb @@ -0,0 +1,47 @@ +module Travis + module Api + module V1 + module Http + class Build + require 'travis/api/v1/http/build/job' + + include Formats, Helpers::Legacy + + attr_reader :build, :commit, :request + + def initialize(build, options = {}) + @build = build + @commit = build.commit + @request = build.request + end + + def data + { + 'id' => build.id, + 'repository_id' => build.repository_id, + 'number' => build.number, + 'config' => build.obfuscated_config.stringify_keys, + 'state' => legacy_build_state(build), + 'result' => legacy_build_result(build), + 'status' => legacy_build_result(build), + 'started_at' => format_date(build.started_at), + 'finished_at' => format_date(build.finished_at), + 'duration' => build.duration, + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'compare_url' => commit.compare_url, + 'event_type' => build.event_type, + 'matrix' => build.matrix.map { |job| Job.new(job).data }, + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/build/job.rb b/vendor/travis-core/lib/travis/api/v1/http/build/job.rb new file mode 100644 index 00000000..e924041d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/build/job.rb @@ -0,0 +1,32 @@ +module Travis + module Api + module V1 + module Http + class Build + class Job + include Formats, Helpers::Legacy + + attr_reader :job + + def initialize(job) + @job = job + end + + def data + { + 'id' => job.id, + 'repository_id' => job.repository_id, + 'number' => job.number, + 'config' => job.obfuscated_config.stringify_keys, + 'result' => legacy_job_result(job), + 'started_at' => format_date(job.started_at), + 'finished_at' => format_date(job.finished_at), + 'allow_failure' => job.allow_failure + } + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/builds.rb b/vendor/travis-core/lib/travis/api/v1/http/builds.rb new file mode 100644 index 00000000..74ca3e3d --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/builds.rb @@ -0,0 +1,38 @@ +module Travis + module Api + module V1 + module Http + class Builds + include Formats, Helpers::Legacy + + attr_reader :builds + + def initialize(builds, options = {}) + @builds = builds + end + + def data + builds.map { |build| build_data(build) } + end + + def build_data(build) + { + 'id' => build.id, + 'repository_id' => build.repository_id, + 'number' => build.number, + 'state' => legacy_build_state(build), + 'result' => legacy_build_result(build), + 'started_at' => format_date(build.started_at), + 'finished_at' => format_date(build.finished_at), + 'duration' => build.duration, + 'commit' => build.commit.commit, + 'branch' => build.commit.branch, + 'message' => build.commit.message, + 'event_type' => build.event_type, + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/hooks.rb b/vendor/travis-core/lib/travis/api/v1/http/hooks.rb new file mode 100644 index 00000000..a35412cb --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/hooks.rb @@ -0,0 +1,34 @@ +module Travis + module Api + module V1 + module Http + class Hooks + attr_reader :repos, :options + + def initialize(repos, options = {}) + @repos = repos + @options = options + end + + def data + repos.map { |repo| repo_data(repo) } + end + + private + + def repo_data(repo) + { + 'uid' => [repo.owner_name, repo.name].join(':'), + 'url' => "https://github.com/#{repo.owner_name}/#{repo.name}", + 'name' => repo.name, + 'owner_name' => repo.owner_name, + 'description' => repo.description, + 'active' => repo.active, + 'private' => repo.private + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/job.rb b/vendor/travis-core/lib/travis/api/v1/http/job.rb new file mode 100644 index 00000000..dd832083 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/job.rb @@ -0,0 +1,44 @@ +module Travis + module Api + module V1 + module Http + class Job + include Formats, Helpers::Legacy + + attr_reader :job, :commit + + def initialize(job, options = {}) + @job = job + @commit = job.commit + end + + def data + { + 'id' => job.id, + 'number' => job.number, + 'config' => job.obfuscated_config.stringify_keys, + 'repository_id' => job.repository_id, + 'build_id' => job.source_id, + 'state' => job.finished? ? 'finished' : job.state.to_s, + 'result' => legacy_job_result(job), + 'status' => legacy_job_result(job), + 'started_at' => format_date(job.started_at), + 'finished_at' => format_date(job.finished_at), + 'log' => job.log_content, + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'compare_url' => commit.compare_url, + 'worker' => job.worker + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/jobs.rb b/vendor/travis-core/lib/travis/api/v1/http/jobs.rb new file mode 100644 index 00000000..3279211b --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/jobs.rb @@ -0,0 +1,34 @@ +module Travis + module Api + module V1 + module Http + class Jobs + include Formats, Helpers::Legacy + + attr_reader :jobs + + def initialize(jobs, options = {}) + @jobs = jobs + end + + def data + jobs.map { |job| job_data(job) } + end + + def job_data(job) + commit = job.commit + { + 'id' => job.id, + 'repository_id' => job.repository_id, + 'number' => job.number, + 'state' => legacy_job_state(job), + 'queue' => job.queue, + 'allow_failure' => job.allow_failure + } + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v1/http/repositories.rb b/vendor/travis-core/lib/travis/api/v1/http/repositories.rb new file mode 100644 index 00000000..efb00792 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/repositories.rb @@ -0,0 +1,37 @@ +module Travis + module Api + module V1 + module Http + class Repositories + include Formats, Helpers::Legacy + + attr_reader :repositories + + def initialize(repositories, options = {}) + @repositories = repositories + end + + def data + repositories.map { |repository| repository_data(repository) } + end + + def repository_data(repository) + { + 'id' => repository.id, + 'slug' => repository.slug, + 'description' => repository.description, + 'last_build_id' => repository.last_build_id, + 'last_build_number' => repository.last_build_number, + 'last_build_status' => legacy_repository_last_build_result(repository), + 'last_build_result' => legacy_repository_last_build_result(repository), + 'last_build_duration' => repository.last_build_duration, + 'last_build_language' => nil, + 'last_build_started_at' => format_date(repository.last_build_started_at), + 'last_build_finished_at' => format_date(repository.last_build_finished_at), + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/repository.rb b/vendor/travis-core/lib/travis/api/v1/http/repository.rb new file mode 100644 index 00000000..dfceebd0 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/repository.rb @@ -0,0 +1,34 @@ +module Travis + module Api + module V1 + module Http + class Repository + include Formats, Helpers::Legacy + + attr_reader :repository, :options + + def initialize(repository, options = {}) + @repository = repository + end + + def data + { + 'id' => repository.id, + 'slug' => repository.slug, + 'description' => repository.description, + 'public_key' => repository.key.public_key, + 'last_build_id' => repository.last_build_id, + 'last_build_number' => repository.last_build_number, + 'last_build_status' => legacy_repository_last_build_result(repository), + 'last_build_result' => legacy_repository_last_build_result(repository), + 'last_build_duration' => repository.last_build_duration, + 'last_build_language' => nil, + 'last_build_started_at' => format_date(repository.last_build_started_at), + 'last_build_finished_at' => format_date(repository.last_build_finished_at), + } + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/http/user.rb b/vendor/travis-core/lib/travis/api/v1/http/user.rb new file mode 100644 index 00000000..833f7533 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/http/user.rb @@ -0,0 +1,31 @@ +module Travis + module Api + module V1 + module Http + class User + include Formats + + attr_reader :user, :options + + def initialize(user, options = {}) + @user = user + @options = options + end + + def data + { + 'login' => user.login, + 'name' => user.name, + 'email' => user.email, + 'gravatar_id' => user.gravatar_id, + 'locale' => user.locale, + 'is_syncing' => user.is_syncing, + 'synced_at' => format_date(user.synced_at) + } + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/api/v1/webhook.rb b/vendor/travis-core/lib/travis/api/v1/webhook.rb new file mode 100644 index 00000000..2b38d28f --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/webhook.rb @@ -0,0 +1,9 @@ +module Travis + module Api + module V1 + module Webhook + require 'travis/api/v1/webhook/build' + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/webhook/build.rb b/vendor/travis-core/lib/travis/api/v1/webhook/build.rb new file mode 100644 index 00000000..ccea2076 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/webhook/build.rb @@ -0,0 +1,27 @@ +module Travis + module Api + module V1 + module Webhook + class Build + require 'travis/api/v1/webhook/build/finished' + + attr_reader :build, :commit, :request, :repository, :options + + def initialize(build, options = {}) + @build = build + @commit = build.commit + @request = build.request + @repository = build.repository + @options = options + end + + private + + def build_url + ["https://#{Travis.config.host}", repository.slug, 'builds', build.id].join('/') + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/webhook/build/finished.rb b/vendor/travis-core/lib/travis/api/v1/webhook/build/finished.rb new file mode 100644 index 00000000..7c49cd95 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/webhook/build/finished.rb @@ -0,0 +1,70 @@ +module Travis + module Api + module V1 + module Webhook + class Build + class Finished < Build + require 'travis/api/v1/webhook/build/finished/job' + + include Formats + + def data + data = { + 'id' => build.id, + 'repository' => repository_data, + 'number' => build.number, + 'config' => build.obfuscated_config.stringify_keys, + 'status' => build.result, + 'result' => build.result, + 'status_message' => result_message, + 'result_message' => result_message, + 'started_at' => format_date(build.started_at), + 'finished_at' => format_date(build.finished_at), + 'duration' => build.duration, + 'build_url' => build_url, + 'commit_id' => commit.id, + 'commit' => commit.commit, + 'base_commit' => request.base_commit, + 'head_commit' => request.head_commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'compare_url' => commit.compare_url, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'matrix' => build.matrix.map { |job| Job.new(job, options).data }, + 'type' => build.event_type, + 'state' => build.state.to_s, + 'pull_request' => build.pull_request?, + 'pull_request_number' => build.pull_request_number, + 'pull_request_title' => build.pull_request_title, + 'tag' => request.tag_name + } + + if commit.pull_request? + data['pull_request_number'] = commit.pull_request_number + end + + data + end + + def repository_data + { + 'id' => repository.id, + 'name' => repository.name, + 'owner_name' => repository.owner_name, + 'url' => repository.url + } + end + + def result_message + @result_message ||= ::Build::ResultMessage.new(build).short + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v1/webhook/build/finished/job.rb b/vendor/travis-core/lib/travis/api/v1/webhook/build/finished/job.rb new file mode 100644 index 00000000..bf0493cc --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v1/webhook/build/finished/job.rb @@ -0,0 +1,50 @@ +module Travis + module Api + module V1 + module Webhook + class Build + class Finished < Build + class Job + include Formats + + attr_reader :job, :commit, :options + + def initialize(job, options = {}) + @job = job + @commit = job.commit + @options = options + end + + def data + data = { + 'id' => job.id, + 'repository_id' => job.repository_id, + 'parent_id' => job.source_id, + 'number' => job.number, + 'state' => job.finished? ? 'finished' : job.state.to_s, + 'config' => job.obfuscated_config, + 'status' => job.result, + 'result' => job.result, + 'commit' => commit.commit, + 'branch' => commit.branch, + 'message' => commit.message, + 'compare_url' => commit.compare_url, + 'committed_at' => format_date(commit.committed_at), + 'author_name' => commit.author_name, + 'author_email' => commit.author_email, + 'committer_name' => commit.committer_name, + 'committer_email' => commit.committer_email, + 'allow_failure' => job.allow_failure + } + data['log'] = job.log_content || '' if options[:include_logs] + data['started_at'] = format_date(job.started_at) if job.started? + data['finished_at'] = format_date(job.finished_at) if job.finished? + data + end + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/api/v2.rb b/vendor/travis-core/lib/travis/api/v2.rb new file mode 100644 index 00000000..e70b19f4 --- /dev/null +++ b/vendor/travis-core/lib/travis/api/v2.rb @@ -0,0 +1,7 @@ +module Travis + module Api + module V2 + end + end +end + diff --git a/vendor/travis-core/lib/travis/chunkifier.rb b/vendor/travis-core/lib/travis/chunkifier.rb new file mode 100644 index 00000000..39176887 --- /dev/null +++ b/vendor/travis-core/lib/travis/chunkifier.rb @@ -0,0 +1,59 @@ +require 'coder/cleaner/simple/encodings' + +module Travis + class Chunkifier < Struct.new(:content, :chunk_size, :options) + include Enumerable + include Coder::Cleaner::Simple::Encodings::UTF_8 + + def initialize(*) + super + + self.options ||= {} + end + + def json? + options[:json] + end + + def length + parts.length + end + + def each(&block) + parts.each(&block) + end + + def parts + @parts ||= split + end + + def split + parts = content.scan(/.{1,#{chunk_split_size}}/m) + chunks = [] + current_chunk = '' + + parts.each do |part| + if too_big?(current_chunk + part) + chunks << current_chunk + current_chunk = part + else + current_chunk << part + end + end + + chunks << current_chunk if current_chunk.length > 0 + + chunks + end + + def chunk_split_size + size = chunk_size / 10 + size == 0 ? 1 : size + end + + def too_big?(current_chunk) + current_chunk = current_chunk.to_json if json? + current_chunk.bytesize > chunk_size + end + end +end diff --git a/vendor/travis-core/lib/travis/commit_command.rb b/vendor/travis-core/lib/travis/commit_command.rb new file mode 100644 index 00000000..2331387f --- /dev/null +++ b/vendor/travis-core/lib/travis/commit_command.rb @@ -0,0 +1,23 @@ +module Travis + class CommitCommand + + def initialize(message) + @message = message.to_s + end + + def skip? + backwards_skip or command == 'skip' + end + + private + attr_reader :message + + def command + message =~ /\[ci(?: |:)([\w ]*)\]/i && $1.downcase + end + + def backwards_skip + message =~ /\[skip\s+ci\]/i && true + end + end +end diff --git a/vendor/travis-core/lib/travis/config/database.rb b/vendor/travis-core/lib/travis/config/database.rb new file mode 100644 index 00000000..6ee04efb --- /dev/null +++ b/vendor/travis-core/lib/travis/config/database.rb @@ -0,0 +1,40 @@ +module Travis + class Config + class Database < Struct.new(:options) + include Helpers + + VARIABLES = { application_name: ENV['DYNO'] || $0, statement_timeout: 10_000 } + DEFAULTS = { adapter: 'postgresql', encoding: 'unicode', variables: VARIABLES } + + def config + config = compact(Url.parse(url).to_h) + config = deep_merge(DEFAULTS, config) unless config.empty? + config[:pool] = pool.to_i if pool + config + end + + private + + def url + env('DATABASE_URL').compact.first + end + + def pool + env('DATABASE_POOL_SIZE', 'DB_POOL').compact.first + end + + def env(*keys) + ENV.values_at(*keys.map { |key| prefix(key) }.flatten) + end + + def prefix(key) + key = [options[:prefix], key].compact.join('_').upcase + ["TRAVIS_#{key}", key] + end + + def options + super || {} + end + end + end +end diff --git a/vendor/travis-core/lib/travis/config/defaults.rb b/vendor/travis-core/lib/travis/config/defaults.rb new file mode 100644 index 00000000..a322f94b --- /dev/null +++ b/vendor/travis-core/lib/travis/config/defaults.rb @@ -0,0 +1,72 @@ +require 'travis/config' + +module Travis + class Config < Hashr + require 'travis/config/database' + require 'travis/config/url' + + HOSTS = { + production: 'travis-ci.org', + staging: 'staging.travis-ci.org', + development: 'localhost:3000' + } + + define host: 'travis-ci.org', + shorten_host: 'trvs.io', + tokens: { internal: 'token' }, + auth: { target_origin: nil }, + assets: { host: HOSTS[Travis.env.to_sym] }, + amqp: { username: 'guest', password: 'guest', host: 'localhost', prefetch: 1 }, + database: { adapter: 'postgresql', database: "travis_#{Travis.env}", encoding: 'unicode', min_messages: 'warning', variables: { statement_timeout: 10_000 } }, + s3: { access_key_id: '', secret_access_key: '' }, + pusher: { app_id: 'app-id', key: 'key', secret: 'secret' }, + sidekiq: { namespace: 'sidekiq', pool_size: 1 }, + smtp: {}, + email: {}, + github: { api_url: 'https://api.github.com', token: 'travisbot-token' }, + async: {}, + notifications: [], # TODO rename to event.handlers + metrics: { reporter: 'librato' }, + logger: { thread_id: true }, + queues: [], + default_queue: 'builds.linux', + jobs: { retry: { after: 60 * 60 * 2, max_attempts: 1, interval: 60 * 5 } }, + queue: { limit: { default: 5, by_owner: {} }, interval: 3 }, + logs: { shards: 1, intervals: { vacuum: 10, regular: 180, force: 3 * 60 * 60 } }, + roles: {}, + archive: {}, + ssl: {}, + redis: { url: 'redis://localhost:6379' }, + repository: { ssl_key: { size: 4096 } }, + repository_filter: { include: [/^rails\/rails/], exclude: [/\/rails$/] }, + encryption: Travis.env == 'development' || Travis.env == 'test' ? { key: 'secret' * 10 } : {}, + sync: { organizations: { repositories_limit: 1000 } }, + states_cache: { memcached_servers: 'localhost:11211' }, + sentry: {}, + services: { find_requests: { max_limit: 100, default_limit: 25 } }, + settings: { timeouts: { defaults: { hard_limit: 50, log_silence: 10 }, maximums: { hard_limit: 180, log_silence: 60 } }, + rate_limit: { defaults: { api_builds: 10 }, maximums: { api_builds: 200 } } }, + endpoints: {}, + oauth2: {} + + default :_access => [:key] + + def initialize(*) + super + load_urls + end + + # Wild monkeypatch to backport travis-config v1.0.x resource url loading. + # Needed for enterprise. + class Docker + def load + config = compact( + database: Database.new.config, + logs_database: Database.new(prefix: :logs).config, + amqp: Url.parse(ENV['TRAVIS_RABBITMQ_URL'] || ENV['RABBITMQ_URL']).to_h.compact, + redis: { url: ENV['TRAVIS_REDIS_URL'] || ENV['REDIS_URL'] }.compact + ) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/config/url.rb b/vendor/travis-core/lib/travis/config/url.rb new file mode 100644 index 00000000..26e3cc6d --- /dev/null +++ b/vendor/travis-core/lib/travis/config/url.rb @@ -0,0 +1,36 @@ +module Travis + class Config + module Url + class Base < Struct.new(:username, :password, :host, :port, :database) + def to_h + Hash[each_pair.to_a] + end + end + + Generic = Class.new(Base) + Postgres = Class.new(Base) + Redis = Class.new(Base) + + class Amqp < Base + alias :vhost :database + + def to_h + super.reject { |key, value| key == :database }.merge(vhost: vhost) + end + end + + class << self + def parse(url) + return Generic.new if url.nil? || url.empty? + uri = URI.parse(url) + const = const_get(camelize(uri.scheme)) + const.new(uri.user, uri.password, uri.host, uri.port, uri.path[1..-1]) + end + + def camelize(string) + string.to_s.split('_').collect(&:capitalize).join + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/engine.rb b/vendor/travis-core/lib/travis/engine.rb new file mode 100644 index 00000000..875a4947 --- /dev/null +++ b/vendor/travis-core/lib/travis/engine.rb @@ -0,0 +1,21 @@ +require 'travis' +require 'rails/engine' + +module Travis + class Engine < Rails::Engine + initializer 'add migrations path' do |app| + # need to insert to both Rails.app.paths and Migrator.migration_paths + # because Rails' stupid rake tasks copy them over before loading the + # engines *unless* multiple rake db tasks are combined (as in rake + # db:create db:migrate). Happens in Rails <= 3.2.2 + paths = [ + Rails.application.paths['db/migrate'], + ActiveRecord::Migrator.migrations_paths + ] + paths.each do |paths| + path = root.join('db/migrate').to_s + paths << path unless paths.include?(path) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/enqueue.rb b/vendor/travis-core/lib/travis/enqueue.rb new file mode 100644 index 00000000..e5ac1d96 --- /dev/null +++ b/vendor/travis-core/lib/travis/enqueue.rb @@ -0,0 +1,6 @@ +module Travis + module Enqueue + require 'travis/enqueue/services' + end +end + diff --git a/vendor/travis-core/lib/travis/enqueue/services.rb b/vendor/travis-core/lib/travis/enqueue/services.rb new file mode 100644 index 00000000..9228aff5 --- /dev/null +++ b/vendor/travis-core/lib/travis/enqueue/services.rb @@ -0,0 +1,14 @@ +module Travis + module Enqueue + module Services + require 'travis/enqueue/services/enqueue_jobs' + + class << self + def register + constants(false).each { |name| const_get(name) } + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb b/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb new file mode 100644 index 00000000..4e52275c --- /dev/null +++ b/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs.rb @@ -0,0 +1,127 @@ +require 'travis/services/base' +require 'travis/support/instrumentation' +require 'travis/support/exceptions/handling' + +module Travis + module Enqueue + module Services + # Finds owners that have queueable jobs and for each owner: + # + # * checks how many jobs can be enqueued + # * finds the oldest N queueable jobs and + # * enqueues them + class EnqueueJobs < Travis::Services::Base + TIMEOUT = 2 + + extend Travis::Instrumentation, Travis::Exceptions::Handling + + require 'travis/enqueue/services/enqueue_jobs/limit' + + register :enqueue_jobs + + def self.run + new.run + end + + def reports + @reports ||= {} + end + + def run + enqueue_all && reports unless disabled? + end + instrument :run + rescues :run, from: Exception, backtrace: false + + def disabled? + Timeout.timeout(TIMEOUT) do + Travis::Features.feature_deactivated?(:job_queueing) + end + rescue Timeout::Error, Redis::TimeoutError => e + Travis.logger.error("[enqueue] Timeout trying to check enqueuing feature flag.") + return false + end + + private + + def enqueue_all + grouped_jobs = jobs.group_by(&:owner) + + Metriks.timer('enqueue.total').time do + grouped_jobs.each do |owner, jobs| + next unless owner + Metriks.timer('enqueue.full_enqueue_per_owner').time do + limit = nil + queueable = nil + Metriks.timer('enqueue.limit_per_owner').time do + Travis.logger.info "About to evaluate jobs for: #{owner.login}." + limit = Limit.new(owner, jobs) + queueable = limit.queueable + end + + Metriks.timer('enqueue.enqueue_per_owner').time do + enqueue(queueable) + end + + Metriks.timer('enqueue.report_per_owner').time do + reports[owner.login] = limit.report + end + end + end + end + end + + def enqueue(jobs) + jobs.each do |job| + Travis.logger.info("enqueueing slug=#{job.repository.slug} job_id=#{job.id}") + Metriks.timer('enqueue.publish_job').time do + publish(job) + end + + Metriks.timer('enqueue.enqueue_job').time do + job.enqueue + end + end + end + + def publish(job) + Metriks.timer('enqueue.publish_job').time do + payload = Travis::Api.data(job, for: 'worker', type: 'Job::Test', version: 'v0') + publisher(job.queue).publish(payload, properties: { type: payload['type'], persistent: true }) + end + end + + def jobs + Metriks.timer('enqueue.fetch_jobs').time do + jobs = Job.includes(:owner).queueable.all + Travis.logger.info "Found #{jobs.size} jobs in total." if jobs.size > 0 + jobs + end + end + + def publisher(queue) + Travis::Amqp::Publisher.builds(queue) + end + + class Instrument < Notification::Instrument + def run_completed + publish(msg: format(target.reports), reports: target.reports) + end + + def format(reports) + reports = Array(reports) + if reports.any? + reports = reports.map do |repo, report| + " #{repo}: #{report.map { |key, value| "#{key}: #{value}" }.join(', ')}" + end + "enqueued:\n#{reports.join("\n")}" + else + 'nothing to enqueue.' + end + end + end + Instrument.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb b/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb new file mode 100644 index 00000000..3d23ef2f --- /dev/null +++ b/vendor/travis-core/lib/travis/enqueue/services/enqueue_jobs/limit.rb @@ -0,0 +1,85 @@ +require 'travis/services/base' +require 'travis/model/job' + +module Travis + module Enqueue + module Services + class EnqueueJobs < Travis::Services::Base + class Limit + attr_reader :owner, :jobs, :config + + def initialize(owner, jobs) + @owner = owner + @jobs = jobs + @config = Travis.config.queue.limit + end + + def queueable + @queueable ||= filter_by_repository(jobs)[0, max_queueable] + end + + def filter_by_repository(jobs) + return jobs unless Travis.config.limit_per_repo_enabled? + queueable_by_repository_id = {} + jobs.reject do |job| + if job.repository.settings.restricts_number_of_builds? + queueable?(job, queueable_by_repository_id, running_by_repository_id) + end + end + end + + def running_by_repository_id + @running_by_repository ||= Hash[running_jobs.group_by(&:repository_id).map {|repository_id, jobs| [repository_id, jobs.size]}] + end + + def queueable?(job, queueable, running) + repository = job.repository_id + queueable[repository] ||= 0 + + runnable_count = queueable[repository] + + (running[repository] || 0) + if runnable_count < job.repository.settings.maximum_number_of_builds + queueable[repository] += 1 + false + else + true + end + end + + def report + { total: jobs.size, running: running, max: max_jobs, queueable: queueable.size } + end + + private + + def running_jobs + @running_jobs ||= Job.owned_by(owner).running + end + + def running + @running ||= Job.owned_by(owner).running.count(:id) + end + + def max_queueable + return config.default if owner.login.nil? + + if unlimited? + 999 + else + queueable = max_jobs - running + queueable < 0 ? 0 : queueable + end + end + + def max_jobs + config.by_owner[owner.login] || config.default + end + + def unlimited? + config.by_owner[owner.login] == -1 + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/errors.rb b/vendor/travis-core/lib/travis/errors.rb new file mode 100644 index 00000000..939590c8 --- /dev/null +++ b/vendor/travis-core/lib/travis/errors.rb @@ -0,0 +1,20 @@ +module Travis + class RepositoryNotFoundError < StandardError + def initialize(params) + details = '' + + if id = params[:repository_id] || params[:id] + details = "with id=#{params[:repository_id] || params[:id]} " + elsif params[:github_id] + details = "with github_id=#{params[:github_id]} " + elsif params.key?(:slug) + details = "with slug=#{params[:slug]} " + elsif params.key?(:name) && params.key?(:owner_name) + details = "with slug=#{params[:name]}/#{params[:owner_name]} " + end + + + super("Repository #{details}could not be found") + end + end +end diff --git a/vendor/travis-core/lib/travis/event.rb b/vendor/travis-core/lib/travis/event.rb new file mode 100644 index 00000000..b4ae76d0 --- /dev/null +++ b/vendor/travis-core/lib/travis/event.rb @@ -0,0 +1,57 @@ +require 'core_ext/module/include' +require 'active_support/core_ext/string/inflections' + +module Travis + + # Event handlers register to events that are issued from state change + # events on the core domain models (such as Request, Build and Job::Test). + # + # Handler registrations are defined in Travis.config so they can be added or + # removed easily for different environments. + # + # Note that Travis::Event#notify accepts an internal event name like + # 'create' (coming from the simple_states implementation in the models) and + # turns it into a namespaced client event name like 'job:test:created'). + # Notification handlers register for and deal with these client event names. + module Event + require 'travis/event/config' + require 'travis/event/handler' + require 'travis/event/subscription' + + SUBSCRIBERS = %w(metrics) + + class << self + include Logging + + def subscriptions + @subscriptions ||= subscribers.map do |name| + name = 'github_status' if name == 'github_commit_status' # TODO compat, remove once configs have been updated + subscription = Subscription.new(name) + subscription if subscription.subscriber + end.compact + end + + def dispatch(event, *args) + subscriptions.each do |subscription| + subscription.notify(event, *args) + end + end + + def subscribers + (SUBSCRIBERS + Travis.config.notifications).uniq + end + end + + def notify(event, *args) + Travis::Event.dispatch(client_event(event, self), self, *args) + end + + protected + + def client_event(event, object) + event = "#{event}ed".gsub(/eded$|eed$/, 'ed') unless [:log, :ready].include?(event) + namespace = object.class.name.underscore.gsub('/', ':').gsub(/travis:model:/, '') + [namespace, event].join(':') + end + end +end diff --git a/vendor/travis-core/lib/travis/event/config.rb b/vendor/travis-core/lib/travis/event/config.rb new file mode 100644 index 00000000..67331a77 --- /dev/null +++ b/vendor/travis-core/lib/travis/event/config.rb @@ -0,0 +1,109 @@ +require 'travis/secure_config' + +module Travis + module Event + class Config + DEFAULTS = { + start: { email: false, webhooks: false, campfire: false, hipchat: false, irc: false, flowdock: false, sqwiggle: false, slack: false, pushover: false }, + success: { email: :change, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always, }, + failure: { email: :always, webhooks: :always, campfire: :always, hipchat: :always, irc: :always, flowdock: :always, sqwiggle: :always, slack: :always, pushover: :always, } + } + + attr_reader :payload, :build, :secure_key, :config + + def initialize(payload, secure_key = nil) + @payload = payload + @build = payload['build'] + @config = build['config'] + @secure_key = secure_key + end + + def enabled?(key) + return !!notifications[key] if notifications.has_key?(key) # TODO this seems inconsistent. what if email: { disabled: true } + [:disabled, :disable].each { |key| return !notifications[key] if notifications.has_key?(key) } # TODO deprecate disabled and disable + true + end + + def send_on?(type, event) + send(:"send_on_#{event}_for?", type) + end + + def send_on_started_for?(type) + config = with_fallbacks(type, :on_start, DEFAULTS[:start][type]) + config == true || config == :always + end + + def send_on_finished_for?(type) + send_on_initial_build? || send_on_success_for?(type) || send_on_failure_for?(type) + end + + def send_on_initial_build? + build['previous_state'].nil? + end + + def send_on_success_for?(type) + !!if build_passed? + config = with_fallbacks(type, :on_success, DEFAULTS[:success][type]) + config == :always || (config == :change && !previous_build_passed?) + end + end + + def send_on_failure_for?(type) + !!if !build_passed? + config = with_fallbacks(type, :on_failure, DEFAULTS[:failure][type]) + config == :always || (config == :change && previous_build_passed?) + end + end + + def build_passed? + build['state'].try(:to_sym) == :passed + end + + def previous_build_passed? + build['previous_state'].try(:to_sym) == :passed + end + + # Fetches config with fallbacks. (notification type > global > default) + # Filters can be configured for each notification type. + # If no rules are configured for the given type, then fall back to the global rules, and then to the defaults. + def with_fallbacks(type, key, default) + config = if (notifications[type] && notifications[type].is_a?(Hash) && notifications[type].has_key?(key)) + # Returns the type config if key is present (notifications: email: [:on_success]) + notifications[type][key] + elsif notifications.has_key?(key) + # Returns the global config if key is present (notifications: [:on_success]) + notifications[key] + else + # Else, returns the given default + default + end + + config.respond_to?(:to_sym) ? config.to_sym : config + end + + # Returns (recipients, urls, channels) for (email, webhooks, irc) + # Notification type config can be nil, true/false, a string, an array of values, + # or a hash containing a key for these values. + def notification_values(type, key) + config = notifications[type] rescue {} + value = config.is_a?(Hash) ? config[key] : config + case value + when Array, String + normalize_array(value) + else + value + end + end + + def notifications + Travis::SecureConfig.decrypt(config.fetch(:notifications, {}), secure_key) + end + + def normalize_array(values) + values = Array(values).compact + values = values.map { |value| value.split(',') if value.is_a?(String) } + values.compact.flatten.map(&:strip).reject(&:blank?) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/event/handler.rb b/vendor/travis-core/lib/travis/event/handler.rb new file mode 100644 index 00000000..177c41ab --- /dev/null +++ b/vendor/travis-core/lib/travis/event/handler.rb @@ -0,0 +1,81 @@ +require 'active_support/core_ext/object/blank' +require 'travis/support/logging' +require 'travis/support/instrumentation' +require 'travis/support/exceptions/handling' + +require 'travis/api' +require 'travis/event/config' +require 'travis/model/build' + +module Travis + module Event + class Handler + require 'travis/event/handler/metrics' + require 'travis/event/handler/trail' + + include Logging + extend Instrumentation, Exceptions::Handling + + class << self + def notify(event, object, data = {}) + payload = Api.data(object, for: 'event', version: 'v0', params: data) if object.is_a?(Build) + handler = new(event, object, data, payload) + handler.notify if handler.handle? + end + end + + attr_reader :event, :object, :data, :payload + + def initialize(event, object, data = {}, payload = {}) + @event = event + @object = object + @data = data + @payload = payload + end + + def notify + handle + end + # TODO disable instrumentation in tests + instrument :notify + rescues :notify, from: Exception + + private + + def config + # TODO: we should decrypt things in tasks, not in event handler, + # secure_key should be passed to the task and then it should + # decrypt the values, which task needs + @config ||= Config.new(payload, secure_key) + end + + def repository + @repository ||= payload['repository'] + end + + def job + @job ||= payload['job'] + end + + def build + @build ||= payload['build'] + end + + def request + @request ||= payload['request'] + end + + def commit + @commit ||= payload['commit'] + end + + def secure_key + object.respond_to?(:repository) ? object.repository.key : nil + end + + def pull_request? + build['pull_request'] + end + end + end +end diff --git a/vendor/travis-core/lib/travis/event/handler/metrics.rb b/vendor/travis-core/lib/travis/event/handler/metrics.rb new file mode 100644 index 00000000..0548c7e8 --- /dev/null +++ b/vendor/travis-core/lib/travis/event/handler/metrics.rb @@ -0,0 +1,50 @@ +require 'travis/support/metrics' + +module Travis + module Event + class Handler + + # Stores metrics about domain events + class Metrics < Handler + EVENTS = /job:test:(started|finished)/ + + def initialize(*) + super + @payload = Api.data(object, type: 'job', for: 'event', version: 'v0', params: data) + end + + def handle? + true + end + + def handle + case event + when 'job:test:started' + events = %W(job.queue.wait_time job.queue.wait_time.#{queue}) + if job['created_at'] && job['started_at'] + meter(events, job['created_at'], job['started_at']) + end + when 'job:test:finished' + events = %W(job.duration job.duration.#{queue}) + if job['started_at'] && job['finished_at'] + meter(events, job['started_at'], job['finished_at']) + end + end + end + + private + + def queue + job['queue'].gsub('.', '-') + end + + def meter(events, started_at, finished_at) + events.each do |event| + Travis::Metrics.meter(event, started_at: started_at, finished_at: finished_at) + end + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/event/handler/trail.rb b/vendor/travis-core/lib/travis/event/handler/trail.rb new file mode 100644 index 00000000..a708f531 --- /dev/null +++ b/vendor/travis-core/lib/travis/event/handler/trail.rb @@ -0,0 +1,42 @@ +module Travis + module Event + class Handler + class Trail < Handler + # EVENTS = [/^((?!event|log|worker).)*$/] # i.e. does not include "log" + EVENTS = /--deactivated--/ + + # attr_reader :data + + # def initialize(*) + # super + # @data = build_data if handle? + # end + + def handle? + false + # Features.feature_active?(:event_trail) + end + + # def handle + # ::Event.create!(:source => object, :repository => repository, :event => event, :data => data) + # end + + # private + + # def repository + # object.is_a?(Repository) ? object : object.repository + # end + + # def build_data + # data = {} + # data[:commit] = object.commit.try(:commit) if object.respond_to?(:commit) + # data[:type] = object.request.try(:event_type) if object.respond_to?(:request) + # data[:number] = object.number if object.respond_to?(:number) + # data[:state] = object.result if object.respond_to?(:state) + # data[:message] = object.message if object.respond_to?(:message) + # data + # end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/event/subscription.rb b/vendor/travis-core/lib/travis/event/subscription.rb new file mode 100644 index 00000000..0897a2ce --- /dev/null +++ b/vendor/travis-core/lib/travis/event/subscription.rb @@ -0,0 +1,61 @@ +require 'active_support/inflector/inflections.rb' +require 'metriks' + +module Travis + module Event + + # Event handlers subscribe to events issued from core models (such + # as Build and Job::Test). + # + # Subscriptions are defined in Travis.config so they can easily be + # added/removed for an environment. + # + # Subscribing classes are supposed to define an EVENTS constant which holds + # a regular expression which will be matched against the event name. + class Subscription + class << self + def register(name, const) + handlers[name.to_sym] = const + end + + def handlers + @handlers ||= {} + end + end + + attr_reader :name + + def initialize(name) + @name = name + end + + def subscriber + self.class.handlers[name.to_sym] || Handler.const_get(name.to_s.camelize, false) + rescue NameError => e + Travis.logger.error "Could not find event handler #{name.inspect}, ignoring." + nil + end + + def patterns + subscriber ? Array(subscriber::EVENTS) : [] + end + + def notify(event, *args) + if matches?(event) + subscriber.notify(event, *args) + increment_counter(event) + end + end + + def matches?(event) + patterns.any? { |patterns| patterns.is_a?(Regexp) ? patterns.match(event) : patterns == event } + end + + def increment_counter(event) + # TODO ask mathias about this metric + metric = "travis.notifications.#{name}.#{event.gsub(/:/, '.')}" + Metriks.meter(metric).mark + end + end + end +end diff --git a/vendor/travis-core/lib/travis/features.rb b/vendor/travis-core/lib/travis/features.rb new file mode 100644 index 00000000..4ed075a1 --- /dev/null +++ b/vendor/travis-core/lib/travis/features.rb @@ -0,0 +1,146 @@ +require 'redis' +require 'rollout' +require 'active_support/deprecation' +require 'active_support/core_ext/module' + +module Travis + # Travis::Features contains methods to handle feature flags. + module Features + class << self + methods = (Rollout.public_instance_methods(false) - [:active?, "active?"]) << {:to => :rollout} + delegate(*methods) + end + + def redis + Travis.redis + end + + def rollout + @rollout ||= ::Rollout.new(redis) + end + + # Returns whether a given feature is enabled either globally or for a given + # repository. + # + # By default, this will return false. + def active?(feature, repository) + feature_active?(feature) or + (rollout.active?(feature, repository.owner) or + repository_active?(feature, repository)) + end + + def activate_repository(feature, repository) + redis.sadd(repository_key(feature), repository_id(repository)) + end + + def deactivate_repository(feature, repository) + redis.srem(repository_key(feature), repository_id(repository)) + end + + # Return whether a given feature is enabled for a repository. + # + # By default, this will return false. + def repository_active?(feature, repository) + redis.sismember(repository_key(feature), repository_id(repository)) + end + + # Return whether a given feature is enabled for a user. + # + # By default, this will return false. + def user_active?(feature, user) + rollout.active?(feature, user) + end + + def activate_all(feature) + redis.del(disabled_key(feature)) + end + + # Return whether a feature is enabled globally. + # + # By default, this will return false. + def feature_active?(feature) + enabled_for_all?(feature) and !feature_inactive?(feature) + end + + # Return whether a feature has been disabled. + # + # This is similar to feature_deactivated?, but with the opposite default. + # + # By default this will return true (ie. disabled). + def feature_inactive?(feature) + redis.get(disabled_key(feature)) != "1" + end + + # Return whether a feature has been disabled. + # + # This is similar to feature_inactive?, but with the opposite default. + # + # By default this will return false (ie not disabled). + def feature_deactivated?(feature) + redis.get(disabled_key(feature)) == '0' + end + + def deactivate_all(feature) + redis.set(disabled_key(feature), 0) + end + + # Return whether a feature has been enabled globally. + # + # By default this will return false. + def enabled_for_all?(feature) + redis.get(enabled_for_all_key(feature)) == '1' + end + + def enable_for_all(feature) + redis.set(enabled_for_all_key(feature), 1) + end + + def disable_for_all(feature) + redis.set(enabled_for_all_key(feature), 0) + end + + def activate_owner(feature, owner) + redis.sadd(owner_key(feature, owner), owner.id) + end + + def deactivate_owner(feature, owner) + redis.srem(owner_key(feature, owner), owner.id) + end + + # Return whether a feature has been enabled for a user. + # + # By default, this return false. + def owner_active?(feature, owner) + redis.sismember(owner_key(feature, owner), owner.id) + end + + extend self + + private + + def key(name) + "feature:#{name}" + end + + def owner_key(feature, owner) + suffix = owner.class.table_name + "#{key(feature)}:#{suffix}" + end + + def repository_key(feature) + "#{key(feature)}:repositories" + end + + def disabled_key(feature) + "#{key(feature)}:disabled" + end + + def enabled_for_all_key(feature) + "#{key(feature)}:disabled" + end + + def repository_id(repository) + repository.respond_to?(:id) ? repository.id : repository.to_i + end + end +end diff --git a/vendor/travis-core/lib/travis/github.rb b/vendor/travis-core/lib/travis/github.rb new file mode 100644 index 00000000..95b5a0ad --- /dev/null +++ b/vendor/travis-core/lib/travis/github.rb @@ -0,0 +1,36 @@ +require 'gh' +require 'core_ext/hash/compact' + +module Travis + module Github + require 'travis/github/services' + + class << self + def setup + GH.set( + client_id: Travis.config.oauth2.client_id, + client_secret: Travis.config.oauth2.client_secret, + user_agent: "Travis-CI/#{TravisCore::VERSION} GH/#{GH::VERSION}", + origin: Travis.config.host, + api_url: Travis.config.github.api_url, + ssl: Travis.config.ssl.to_h.merge(Travis.config.github.ssl || {}).to_h.compact + ) + end + + def authenticated(user, &block) + fail "we don't have a github token for #{user.inspect}" if user.github_oauth_token.blank? + GH.with(:token => user.github_oauth_token, &block) + end + + # TODO: Maybe this should move to gh? + def scopes_for(token) + token = token.github_oauth_token if token.respond_to? :github_oauth_token + scopes = GH.with(token: token.to_s) { GH.head('user') }.headers['x-oauth-scopes'] if token.present? + scopes &&= scopes.gsub(/\s/,'').split(',') + Array(scopes).sort + rescue GH::Error + [] + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/education.rb b/vendor/travis-core/lib/travis/github/education.rb new file mode 100644 index 00000000..765ecf30 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/education.rb @@ -0,0 +1,45 @@ +require 'timeout' +require 'json' + +module Travis + module Github + class Education < Struct.new(:github_oauth_token) + def self.active?(owner) + if Travis::Features.feature_active?(:education) || Travis::Features.owner_active?(:education, owner) + owner.education? if owner.respond_to? :education? + end + end + + def self.education_queue?(owner) + # this method is here so it can be overridden with subscription logic + active?(owner) + end + + include Travis::Logging + + def student? + data['student'] + end + + def data + @data ||= fetch + end + + def fetch + Timeout::timeout(timeout) do + remote = GH::Remote.new + remote.setup('https://education.github.com/api', token: github_oauth_token) + response = remote.fetch_resource('/user') + JSON.parse(response.body) + end + rescue GH::Error, JSON::ParserError, Timeout::Error => e + log_exception(e) unless e.is_a? GH::Error and e.info[:response_status] == 401 + {} + end + + def timeout + Travis.config.education_endpoint_timeout || 2 + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services.rb b/vendor/travis-core/lib/travis/github/services.rb new file mode 100644 index 00000000..fa408c79 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services.rb @@ -0,0 +1,18 @@ +module Travis + module Github + module Services + require 'travis/github/services/fetch_config' + require 'travis/github/services/find_or_create_org' + require 'travis/github/services/find_or_create_repo' + require 'travis/github/services/find_or_create_user' + require 'travis/github/services/set_hook' + require 'travis/github/services/sync_user' + + class << self + def register + constants(false).each { |name| const_get(name) } + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/fetch_config.rb b/vendor/travis-core/lib/travis/github/services/fetch_config.rb new file mode 100644 index 00000000..3d16aa15 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/fetch_config.rb @@ -0,0 +1,90 @@ +require 'gh' +require 'yaml' +require 'active_support/core_ext/class/attribute' +require 'travis/support/logging' +require 'travis/support/instrumentation' +require 'travis/services/base' + +module Travis + module Github + module Services + # encapsulates fetching a .travis.yml from a given commit's config_url + class FetchConfig < Travis::Services::Base + include Logging + extend Instrumentation + + register :github_fetch_config + + def run + config = retrying(3) { filter(parse(fetch)) } + config || Travis.logger.warn("[request:fetch_config] Empty config for request id=#{request.id} config_url=#{config_url.inspect}") + rescue GH::Error => e + if e.info[:response_status] == 404 + { '.result' => 'not_found' } + else + { '.result' => 'server_error' } + end + end + instrument :run + + def request + params[:request] + end + + def config_url + request.config_url + end + + private + + def fetch + content = GH[config_url]['content'] + Travis.logger.warn("[request:fetch_config] Empty content for #{config_url}") if content.nil? + content = content.to_s.unpack('m').first + Travis.logger.warn("[request:fetch_config] Empty unpacked content for #{config_url}, content was #{content.inspect}") if content.nil? + nbsp = "\xC2\xA0".force_encoding("binary") + content = content.gsub(/^(#{nbsp})+/) { |match| match.gsub(nbsp, " ") } + + content + end + + def parse(yaml) + YAML.load(yaml).merge('.result' => 'configured') + rescue StandardError, Psych::SyntaxError => e + error "[request:fetch_config] Error parsing .travis.yml for #{config_url}: #{e.message}" + { + '.result' => 'parse_error', + '.result_message' => e.is_a?(Psych::SyntaxError) ? e.message.split(": ").last : e.message + } + end + + def filter(config) + unless Travis::Features.active?(:template_selection, request.repository) + config = config.to_h.except('dist').except('group') + end + + config + end + + def retrying(times) + count, result = 0, nil + until result || count > times + result = yield + count += 1 + Travis.logger.warn("[request:fetch_config] Retrying to fetch config for #{config_url}") unless result + end + result + end + + class Instrument < Notification::Instrument + def run_completed + # TODO exctract something like Url.strip_secrets + config_url = target.config_url.gsub(/(token|secret)=\w*/) { "#{$1}=[secure]" } + publish(msg: "#{config_url}", url: config_url, result: result) + end + end + Instrument.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/find_or_create_org.rb b/vendor/travis-core/lib/travis/github/services/find_or_create_org.rb new file mode 100644 index 00000000..7bb4e748 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/find_or_create_org.rb @@ -0,0 +1,80 @@ +require 'gh' +require 'travis/services/base' +require 'travis/model/organization' +require 'travis/model/repository' +require 'travis/model/user' + +module Travis + module Github + module Services + class FindOrCreateOrg < Travis::Services::Base + register :github_find_or_create_org + + def run + find || create + end + + private + + def find + ::Organization.where(:github_id => params[:github_id]).first.tap do |organization| + if organization + ActiveRecord::Base.transaction do + login = params[:login] || data['login'] + if organization.login != login + Repository.where(owner_name: organization.login). + update_all(owner_name: login) + organization.update_attributes(login: login) + end + + nullify_logins(organization.github_id, organization.login) + end + end + end + end + + def nullify_logins(github_id, login) + users = User.where(["login = ?", login]) + if users.exists? + Travis.logger.info("About to nullify login (#{login}) for users: #{users.map(&:id).join(', ')}") + users.update_all(login: nil) + end + + organizations = Organization.where(["github_id <> ? AND login = ?", github_id, login]) + if organizations.exists? + Travis.logger.info("About to nullify login (#{login}) for organizations: #{organizations.map(&:id).join(', ')}") + organizations.update_all(login: nil) + end + end + + def create + organization = Organization.create!( + :name => data['name'], + :login => data['login'], + :github_id => data['id'], + :email => data['email'], + :location => data['location'], + :avatar_url => data['_links'] && data['_links']['avatar'].try(:fetch, 'href'), + :company => data['company'], + :homepage => data['_links'] && data['_links']['blog'].try(:fetch, 'href') + ) + + nullify_logins(organization.github_id, organization.login) + + organization + rescue ActiveRecord::RecordNotUnique + find + end + + def avatar_url(github_data) + href = github_data.try(:fetch, 'href') + href ? href[/^(https:\/\/[\w\.\/]*)/, 1] : nil + end + + def data + @data ||= GH["organizations/#{params[:github_id]}"] || raise(Travis::GithubApiError) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/find_or_create_repo.rb b/vendor/travis-core/lib/travis/github/services/find_or_create_repo.rb new file mode 100644 index 00000000..c85835cf --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/find_or_create_repo.rb @@ -0,0 +1,44 @@ +module Travis + module Github + module Services + class FindOrCreateRepo < Travis::Services::Base + register :github_find_or_create_repo + + def run + repo = find || create + repo.update_attributes(params) + repo + end + + private + + def find + unless params[:github_id] + message = "No github_id passed to FindOrCreateRepo#find, params: #{params.inspect}" + ActiveSupport::Deprecation.warn(message) + Travis.logger.info(message) + end + + query = if params[:github_id] + { github_id: params[:github_id] } + else + { owner_name: params[:owner_name], name: params[:name] } + end + + run_service(:find_repo, query) + end + + def create + unless params[:github_id] + message = "No github_id passed to FindOrCreateRepo#find, params: #{params.inspect}" + ActiveSupport::Deprecation.warn(message) + Travis.logger.info(message) + end + Repository.create!(:owner_name => params[:owner_name], :name => params[:name], github_id: params[:github_id]) + rescue ActiveRecord::RecordNotUnique + find + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/find_or_create_user.rb b/vendor/travis-core/lib/travis/github/services/find_or_create_user.rb new file mode 100644 index 00000000..839eb5e8 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/find_or_create_user.rb @@ -0,0 +1,66 @@ +require 'gh' +require 'travis/model/repository' +require 'travis/model/user' +require 'travis/model/user/renaming' +require 'travis/services/base' + +module Travis + module Github + module Services + class FindOrCreateUser < Travis::Services::Base + register :github_find_or_create_user + + def run + find || create + end + + private + + include ::User::Renaming + + def find + ::User.where(github_id: params[:github_id]).first.tap do |user| + if user + ActiveRecord::Base.transaction do + login = params[:login] || data['login'] + if user.login != login + Travis.logger.info("Changing # login: current=\"#{user.login}\", new=\"#{login}\" (FindOrCreateUser), data: #{data.inspect}") + rename_repos_owner(user.login, login) + user.update_attributes(login: login) + end + end + + nullify_logins(user.github_id, user.login) + end + end + end + + def create + user = User.create!( + :name => data['name'], + :login => data['login'], + :email => data['email'], + :github_id => data['id'], + :gravatar_id => data['gravatar_id'] + ) + + nullify_logins(user.github_id, user.login) + + user + rescue ActiveRecord::RecordNotUnique + find + end + + def data + @data ||= fetch_data + end + + def fetch_data + data = GH["user/#{params[:github_id]}"] || raise(Travis::GithubApiError) + Travis.logger.info("Fetching data for github_id=#{params[:github_id]} (FindOrCreateUser), data: #{data.inspect}") + data + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/set_hook.rb b/vendor/travis-core/lib/travis/github/services/set_hook.rb new file mode 100644 index 00000000..338e01fc --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/set_hook.rb @@ -0,0 +1,67 @@ +require 'travis/github' +require 'travis/services/base' + +module Travis + module Github + module Services + class SetHook < Travis::Services::Base + EVENTS = [:push, :pull_request, :issue_comment, :public, :member] + + register :github_set_hook + + def run + Github.authenticated(current_user) do + update + end + end + + private + + def repo + @repo ||= run_service(:find_repo, id: params[:id]) + end + + def active? + params[:active] + end + + def hook + @hook ||= find || create + end + + def update + GH.patch(hook_url, payload) unless hook['active'] == active? + end + + def find + GH[hooks_url].detect { |hook| hook['name'] == 'travis' && hook['config']['domain'] == domain } + end + + def create + GH.post(hooks_url, payload) + end + + def payload + { + :name => 'travis', + :events => EVENTS, + :active => active?, + :config => { :user => current_user.login, :token => current_user.tokens.first.token, :domain => domain } + } + end + + def hooks_url + "repos/#{repo.slug}/hooks" + end + + def hook_url + hook['_links']['self']['href'] + end + + def domain + Travis.config.service_hook_url || '' + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/sync_user.rb b/vendor/travis-core/lib/travis/github/services/sync_user.rb new file mode 100644 index 00000000..1c5fda57 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/sync_user.rb @@ -0,0 +1,74 @@ +require 'metriks' +require 'travis/mailer/user_mailer' +require 'travis/services/base' + +module Travis + module Github + module Services + class SyncUser < Travis::Services::Base + require 'travis/github/services/sync_user/organizations' + require 'travis/github/services/sync_user/repositories' + require 'travis/github/services/sync_user/repository' + require 'travis/github/services/sync_user/reset_token' + require 'travis/github/services/sync_user/user_info' + + register :github_sync_user + + def run + new_user? do + syncing do + # if Time.now.utc.tuesday? && Travis::Features.feature_active?("reset_token_in_sync") + # ResetToken.new(user).run + # end + UserInfo.new(user).run + Organizations.new(user).run + Repositories.new(user).run + end + end + ensure + user.update_column(:is_syncing, false) + end + + def user + # TODO check that clients are only passing the id + @user ||= current_user || User.find(params[:id]) + end + + def new_user? + new_user = user.synced_at.nil? && user.created_at > 48.hours.ago.utc + + yield if block_given? + + if new_user and Travis.config.welcome_email + send_welcome_email + end + end + + def send_welcome_email + return unless user.email.present? + UserMailer.welcome_email(user).deliver + logger.info("Sent welcome email to #{user.login}") + Metriks.meter('travis.welcome.email').mark + end + + private + + def syncing + unless user.github_oauth_token? + logger.warn "user sync for #{user.login} (id:#{user.id}) was cancelled as the user doesn't have a token" + return + end + user.update_column(:is_syncing, true) + result = yield + user.update_column(:synced_at, Time.now) + result + rescue GH::TokenInvalid => e + logger.warn "user sync for #{user.login} (id:#{user.id}) failed as the token was invalid, dropping the token" + user.update_column(:github_oauth_token, nil) + ensure + user.update_column(:is_syncing, false) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/sync_user/organizations.rb b/vendor/travis-core/lib/travis/github/services/sync_user/organizations.rb new file mode 100644 index 00000000..f04caf33 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/sync_user/organizations.rb @@ -0,0 +1,143 @@ +require 'gh' + +module Travis + module Github + module Services + class SyncUser < Travis::Services::Base + class Organizations + class Filter + attr_reader :data, :limit + def initialize(data, options = {}) + @data = data || {} + @limit = options[:repositories_limit] || 1000 + end + + def allow? + repositories_count < limit + end + + def repositories_count + # I was not sure how to handle the case where we don't get the + # sufficient amount of data here and this seems the best answer, + # that way we will not get orgs siltently ignored + data['public_repositories'] || 0 + end + end + + class << self + def cancel_memberships(user, orgs) + user.memberships.where(:organization_id => orgs.map(&:id)).delete_all + end + end + + extend Travis::Instrumentation + include Travis::Logging + + attr_reader :user, :data + + def initialize(user) + @user = user + end + + def run + with_github do + { :synced => create_or_update, :removed => remove } + end + end + instrument :run + + private + + def create_or_update + fetch_and_filter.map do |data| + org = create_or_update_org(data) + user.organizations << org unless user.organizations.include?(org) + org + end + end + + def remove + orgs = user.organizations.reject { |org| github_ids.include?(org.github_id) } + self.class.cancel_memberships(user, orgs) + orgs + end + + def fetch + @data ||= GH['user/orgs'].to_a + end + instrument :fetch, :level => :debug + + def github_ids + @github_ids ||= data.map { |org| org['id'] } + end + + def with_github(&block) + # TODO in_parallel should return the block's result in a future version + result = nil + GH.with(:token => user.github_oauth_token) do + # GH.in_parallel do + result = yield + # end + end + result + end + + def fetch_and_filter + fetch.map do |data| + fetch_resource("organizations/#{data['id']}") + end.find_all do |data| + options = Travis.config.sync.organizations || {} + Filter.new(data, options).allow? + end + end + + def fetch_resource(resource) + GH[resource] # TODO should be: ?type=#{self.class.type} but GitHub doesn't work as documented + rescue GH::Error => e + log_exception(e) + end + + def create_or_update_org(data) + org = Organization.find_or_create_by_github_id(data['id']) + org.update_attributes!({ + :name => data['name'], + :login => data['login'], + :email => data['email'], + :avatar_url => avatar_url(data['_links']['avatar']), + :location => data['location'], + :homepage => data['_links']['blog'].try(:fetch, 'href'), + :company => data['company'] + }) + org + end + + def avatar_url(github_data) + href = github_data.try(:fetch, 'href') + href ? href[/^(https:\/\/[\w\.\/]*)/, 1] : nil + end + + class Instrument < Notification::Instrument + def run_completed + format = lambda do |orgs| + orgs.map { |org| { id: org.id, login: org.login } } + end + + publish( + msg: %(for #), + result: { synced: format.call(result[:synced]), removed: format.call(result[:removed]) } + ) + end + + def fetch_completed + publish( + msg: %(for #), + result: result + ) + end + end + Instrument.attach_to(self) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/sync_user/repositories.rb b/vendor/travis-core/lib/travis/github/services/sync_user/repositories.rb new file mode 100644 index 00000000..294f670e --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/sync_user/repositories.rb @@ -0,0 +1,140 @@ +require 'active_support/core_ext/class/attribute' + +module Travis + module Github + module Services + class SyncUser < Travis::Services::Base + # Fetches all repositories from Github which are in /user/repos or any of the user's + # orgs/[name]/repos. Creates or updates existing repositories on our side and adds + # it to the user's permissions. Also removes existing permissions for repositories + # which are not in the received Github data. NOTE that this does *not* delete any + # repositories because we do not know if the repository was deleted or renamed + # on Github's side. + class Repositories + extend Travis::Instrumentation + include Travis::Logging + + class_attribute :types + self.types = [:public] + + class << self + # TODO backwards compat, remove once all apps use `types=` + def type=(types) + self.types = Array.wrap(types).map(&:to_s).join(',').split(',').map(&:to_sym) + end + + def include?(type) + self.types.include?(type) + end + end + + attr_reader :user, :resources, :data + + def initialize(user) + @user = user + @resources = ['user/repos'] + user.organizations.map { |org| "orgs/#{org.login}/repos" } + end + + def run + with_github do + { :synced => create_or_update, :removed => remove } + end + end + instrument :run + + private + + def create_or_update + data.map do |repository| + Repository.new(user, repository).run + end + end + + def remove + repos = user.repositories.reject { |repo| slugs.include?(repo.slug) } + Repository.unpermit_all(user, repos) + repos + end + + # we have to filter these ourselves because the github api is broken for this + def data + @data ||= filter_duplicates(filter_based_on_repo_permission) + end + + def filter_based_on_repo_permission + fetch.select { |repo| self.class.include?(repo['private'] ? :private : :public) } + end + + def filter_duplicates(repositories) + repositories.each_with_object([]) do |repository, filtered_list| + unless in_filtered_list?(filtered_list, repository) + filtered_list.push(repository) + end + end + end + + def in_filtered_list?(filtered_list, other_repository) + filtered_list.any? do |existing_repository| + same_repository_with_admin?(existing_repository, other_repository) + end + end + + def same_repository_with_admin?(existing_repository, other_repository) + existing_repository['owner']['login'] == other_repository['owner']['login'] and + existing_repository['name'] == other_repository['name'] and + existing_repository['permissions']['admin'] == true + end + + def slugs + @slugs ||= data.map { |repo| "#{repo['owner']['login']}/#{repo['name']}" } + end + + def fetch + resources.map { |resource| fetch_resource(resource) }.map(&:to_a).flatten.compact + end + instrument :fetch, :level => :debug + + def fetch_resource(resource) + GH[resource] # TODO should be: ?type=#{self.class.type} but GitHub doesn't work as documented + rescue GH::Error => e + log_exception(e) + end + + def with_github(&block) + # TODO in_parallel should return the block's result in a future version + result = nil + GH.with(:token => user.github_oauth_token) do + # GH.in_parallel do + result = yield + # end + end + result + end + + class Instrument < Notification::Instrument + def run_completed + format = lambda do |repos| + repos.map { |repo| { id: repo.id, owner: repo.owner_name, name: repo.name } } + end + + publish( + msg: %(for #), + resources: target.resources, + result: { synced: format.call(result[:synced]), removed: format.call(result[:removed]) } + ) + end + + def fetch_completed + publish( + msg: %(for #), + resources: target.resources, + result: result + ) + end + end + Instrument.attach_to(self) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/sync_user/repository.rb b/vendor/travis-core/lib/travis/github/services/sync_user/repository.rb new file mode 100644 index 00000000..1636d0f5 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/sync_user/repository.rb @@ -0,0 +1,140 @@ +module Travis + module Github + module Services + class SyncUser < Travis::Services::Base + class Repository + class << self + def unpermit_all(user, repositories) + user.permissions.where(:repository_id => repositories.map(&:id)).delete_all unless repositories.empty? + end + end + + attr_reader :user, :data, :repo + + def initialize(user, data) + @user = user + @data = data + end + + def run + @repo = find || create + update + if permission + sync_permissions + elsif permit? + permit + end + repo + end + + private + + def find + ::Repository.where(:github_id => github_id).first + end + + def create + if Travis::Features.enabled_for_all?(:sync_repo_owner) + ::Repository.create!(:owner => owner, :owner_name => owner_name, :name => name, github_id: github_id) + else + ::Repository.create!(:owner_name => owner_name, :name => name, github_id: github_id) + end + end + # instrument :create, :level => :debug + + def permission + @permission ||= user.permissions.where(:repository_id => repo.id).first + end + + def sync_permissions + if permit? + permission.update_attributes!(permission_data) + else + permission.destroy + end + end + + def permit? + push_access? || admin_access? || repo.private? + end + + def permit + user.permissions.create!({ + :user => user, + :repository => repo + }.merge(permission_data)) + end + # instrument :permit, :level => :debug + + def update + if Travis::Features.enabled_for_all?(:sync_repo_owner) + repo.update_attributes!({ + owner: owner, + github_id: data['id'], + private: data['private'], + description: data['description'], + url: data['homepage'], + default_branch: data['default_branch'], + github_language: data['language'], + name: name, + owner_name: owner_name + }) + else + repo.update_attributes!({ + github_id: data['id'], + private: data['private'], + description: data['description'], + url: data['homepage'], + default_branch: data['default_branch'], + github_language: data['language'], + name: name, + owner_name: owner_name + }) + end + rescue ActiveRecord::RecordInvalid + # ignore for now. this seems to happen when multiple syncs (i.e. user sign + # in requests are running in parallel? + rescue GH::Error(response_status: 404) => e + Travis.logger.warn "[github][services][user_sync] GitHub info was not available for #{repo.owner_name}/#{repo.name}: #{e.inspect}" + end + + def owner + @owner ||= owner_type.constantize.find_by_github_id(owner_id) + end + + def owner_id + data['owner']['id'] + end + + def owner_type + data['owner']['type'] + end + + def owner_name + data['owner']['login'] + end + + def name + data['name'] + end + + def github_id + data['id'] + end + + def permission_data + data['permissions'] + end + + def push_access? + permission_data['push'] + end + + def admin_access? + permission_data['admin'] + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/sync_user/reset_token.rb b/vendor/travis-core/lib/travis/github/services/sync_user/reset_token.rb new file mode 100644 index 00000000..cc09ae26 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/sync_user/reset_token.rb @@ -0,0 +1,36 @@ +require "gh" + +module Travis + module Github + module Services + class SyncUser < Travis::Services::Base + class ResetToken + def initialize(user, config = Travis.config.oauth2.to_h, gh = nil) + @user = user + @config = config + @gh = gh || GH.with(username: @config.client_id, password: @config.client_secret) + end + + def run + token = new_token + @user.update_attributes!(github_oauth_token: token) if token + end + + private + + def new_token + @new_token ||= @gh.post("/applications/#{client_id}/tokens/#{@user.github_oauth_token}", {})["token"] + end + + def client_id + @config.client_id + end + + def client_secret + @config.client_secret + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/github/services/sync_user/user_info.rb b/vendor/travis-core/lib/travis/github/services/sync_user/user_info.rb new file mode 100644 index 00000000..c3be5399 --- /dev/null +++ b/vendor/travis-core/lib/travis/github/services/sync_user/user_info.rb @@ -0,0 +1,90 @@ +require 'gh' +require 'travis/github/education' + +module Travis + module Github + module Services + class SyncUser < Travis::Services::Base + class UserInfo + attr_reader :user, :gh + + def initialize(user, gh = Github.authenticated(user)) + @user, @gh = user, gh + end + + def run + if user.github_id != user_info['id'].to_i + raise "Updating # failed, github_id differs. github_id on user: #{user.github_id}, github_id from data: #{user_info['id']}" + end + if user.login != login + Travis.logger.info("Changing # login: current=\"#{user.login}\", new=\"#{login}\" (UserInfo), data: #{user_info.inspect}") + end + if user.email != email + Travis.logger.info("Changing # email: current=\"#{user.email}\", new=\"#{email}\" (UserInfo)") + end + + user.update_attributes!(name: name, login: login, gravatar_id: gravatar_id, email: email, education: education) + emails = verified_emails + emails << email unless emails.include? email + emails.each { |e| user.emails.find_or_create_by_email!(e) } + end + + def education + if Travis::Features.feature_active?(:education_data_sync) || Travis::Features.owner_active?(:education_data_sync, user) + Education.new(user.github_oauth_token).student? + end + end + + def name + user_info['name'] + end + + def login + user_info.fetch('login') + end + + def gravatar_id + user_info['gravatar_id'] + end + + def email + user_info['email'].presence || primary_email || verified_email || user.email.presence || first_email + end + + def verified_emails + emails.select { |e| e["verified"] }.map { |e| e['email'] } + end + + private + + def emails + return [] unless user.github_scopes.include? 'user' or user.github_scopes.include? 'user:email' + @emails ||= gh['user/emails'].to_a + end + + def first_email + emails.first.try(:[], 'email') + end + + def primary_email + emails.detect { |e| e["primary"] }.try(:[], 'email') + end + + def verified_email + verified_emails.first + end + + def user_info + @user_info ||= begin + data = gh['user'].to_hash + if user.login != data['login'] + Travis.logger.info("Fetching data for github_id=#{user.github_id} (UserInfo), data: #{data.inspect}") + end + data + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/logs.rb b/vendor/travis-core/lib/travis/logs.rb new file mode 100644 index 00000000..89dfa53b --- /dev/null +++ b/vendor/travis-core/lib/travis/logs.rb @@ -0,0 +1,6 @@ +module Travis + module Logs + autoload :Services, 'travis/logs/services' + end +end + diff --git a/vendor/travis-core/lib/travis/logs/services.rb b/vendor/travis-core/lib/travis/logs/services.rb new file mode 100644 index 00000000..f235cf5a --- /dev/null +++ b/vendor/travis-core/lib/travis/logs/services.rb @@ -0,0 +1,17 @@ +module Travis + module Logs + module Services + autoload :Aggregate, 'travis/logs/services/aggregate' + autoload :Archive, 'travis/logs/services/archive' + autoload :Receive, 'travis/logs/services/receive' + + class << self + def register + constants(false).each { |name| const_get(name) } + end + end + end + end +end + + diff --git a/vendor/travis-core/lib/travis/logs/services/aggregate.rb b/vendor/travis-core/lib/travis/logs/services/aggregate.rb new file mode 100644 index 00000000..944f2f9d --- /dev/null +++ b/vendor/travis-core/lib/travis/logs/services/aggregate.rb @@ -0,0 +1,96 @@ +require 'active_support/core_ext/string/filters' +require 'travis/features' +require 'travis/model/log' +require 'travis/model/log/part' +require 'travis/services/base' + +module Travis + module Logs + module Services + class Aggregate < Travis::Services::Base + register :logs_aggregate + + AGGREGATE_UPDATE_SQL = <<-sql.squish + UPDATE logs + SET aggregated_at = ?, + content = (COALESCE(content, '') || (#{Log::AGGREGATE_PARTS_SELECT_SQL})) + WHERE logs.id = ? + sql + + AGGREGATEABLE_SELECT_SQL = <<-sql.squish + SELECT DISTINCT log_id + FROM log_parts + WHERE created_at <= NOW() - interval '? seconds' AND final = ? + OR created_at <= NOW() - interval '? seconds' + sql + + def run + return unless active? + aggregateable_ids.each do |id| + transaction do + aggregate(id) + vacuum(id) + notify(id) + end + end + end + + private + + def active? + Travis::Features.feature_active?(:log_aggregation) + end + + def aggregate(id) + meter('logs.aggregate') do + connection.execute(sanitize_sql([AGGREGATE_UPDATE_SQL, Time.now, id, id])) + end + end + + def vacuum(id) + meter('logs.vacuum') do + Log::Part.delete_all(log_id: id) + end + end + + def notify(id) + Log.find(id).notify('aggregated') + rescue ActiveRecord::RecordNotFound + puts "[warn] could not find a log with the id #{id}" + end + + def aggregateable_ids + Log::Part.connection.select_values(query).map { |id| id.nil? ? id : id.to_i } + end + + def query + Log::Part.send(:sanitize_sql, [AGGREGATEABLE_SELECT_SQL, intervals[:regular], true, intervals[:force]]) + end + + def intervals + Travis.config.logs.intervals + end + + def transaction(&block) + ActiveRecord::Base.transaction(&block) + rescue ActiveRecord::ActiveRecordError => e + # puts e.message, e.backtrace + Travis::Exceptions.handle(e) + end + + def meter(name, &block) + Metriks.timer(name).time(&block) + end + + def connection + Log::Part.connection + end + + def sanitize_sql(*args) + Log::Part.send(:sanitize_sql, *args) + end + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/logs/services/archive.rb b/vendor/travis-core/lib/travis/logs/services/archive.rb new file mode 100644 index 00000000..98890b31 --- /dev/null +++ b/vendor/travis-core/lib/travis/logs/services/archive.rb @@ -0,0 +1,167 @@ +begin + require 'aws/s3' +rescue LoadError => e +end +require 'uri' +require 'active_support/core_ext/hash/slice' +require 'faraday' +require 'travis/support/instrumentation' +require 'travis/notification/instrument' +require 'travis/services/base' + +module Travis + class S3 + class << self + def setup + AWS.config(Travis.config.s3.to_h.slice(:access_key_id, :secret_access_key)) + end + end + + attr_reader :s3, :url + + def initialize(url) + @s3 = AWS::S3.new + @url = url + end + + def store(data) + object.write(data, content_type: 'text/plain', acl: :public_read) + end + + def object + @object ||= bucket.objects[URI.parse(url).path[1..-1]] + end + + def bucket + @bucket ||= s3.buckets[URI.parse(url).host] + end + end + + module Logs + module Services + class Archive < Travis::Services::Base + class FetchFailed < StandardError + def initialize(source_url, status, message) + super("Could not retrieve #{source_url}. Response status: #{status}, message: #{message}") + end + end + + class VerificationFailed < StandardError + def initialize(source_url, target_url, expected, actual) + super("Expected #{target_url} (from: #{source_url}) to have the content length #{expected.inspect}, but had #{actual.inspect}") + end + end + + extend Travis::Instrumentation + + register :archive_log + + attr_reader :log + + def run + fetch + store + verify + report + end + instrument :run + + def source_url + "https://#{hostname('api')}/logs/#{params[:id]}.txt" + end + + def report_url + "https://#{hostname('api')}/logs/#{params[:id]}" + end + + def target_url + "http://#{hostname('archive')}/jobs/#{params[:job_id]}/log.txt" + end + + private + + def fetch + retrying(:fetch) do + response = request(:get, source_url) + if response.status == 200 + @log = response.body.to_s + else + raise(FetchFailed.new(source_url, response.status, response.body.to_s)) + end + end + end + + def store + retrying(:store) do + S3.setup + s3.store(log) + end + end + + def verify + retrying(:verify) do + expected = log.bytesize + actual = request(:head, target_url).headers['content-length'].try(:to_i) + raise VerificationFailed.new(target_url, source_url, expected, actual) unless expected == actual + end + end + + def report + retrying(:report) do + request(:put, report_url, { archived_at: Time.now, archive_verified: true }, token: Travis.config.tokens.internal) + end + end + + def request(method, url, params = nil, headers = nil, &block) + http.send(*[method, url, params, headers].compact, &block) + rescue Faraday::Error => e + puts "Exception while trying to #{method.inspect}: #{source_url}:" + puts e.message, e.backtrace + raise e + end + + def http + Faraday.new(ssl: Travis.config.ssl.to_h.compact) do |f| + f.request :url_encoded + f.adapter :net_http + end + end + + def s3 + S3.new(target_url) + end + + def hostname(name) + "#{name}#{'-staging' if Travis.env == 'staging'}.#{Travis.config.host.split('.')[-2, 2].join('.')}" + end + + def retrying(header, times = 5) + yield + rescue => e + count ||= 0 + if times > (count += 1) + puts "[#{header}] retry #{count} because: #{e.message}" + Travis::Instrumentation.meter("#{self.class.name.underscore.gsub("/", ".")}.retries.#{header}") + sleep count * 3 unless params[:no_sleep] + retry + else + raise + end + end + + class Instrument < Notification::Instrument + def run_completed + publish( + msg: "for (to: #{target.target_url})", + source_url: target.source_url, + target_url: target.target_url, + object_type: 'Log', + object_id: target.params[:id] + ) + end + end + Instrument.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/logs/services/receive.rb b/vendor/travis-core/lib/travis/logs/services/receive.rb new file mode 100644 index 00000000..8f56cc80 --- /dev/null +++ b/vendor/travis-core/lib/travis/logs/services/receive.rb @@ -0,0 +1,79 @@ +require 'coder' +require 'metriks' +require 'travis/model/log/part' +require 'travis/model/job/test' +require 'travis/services/base' + +module Travis + module Logs + module Services + class Receive < Travis::Services::Base + # TODO remove this once we know aggregation works fine and the worker passes a :final flag + FINAL = 'Done. Build script exited with:' + + register :logs_receive + + def run + create_part + notify + end + + private + + def create_part + measure('logs.update') do + Travis.logger.warn "[warn] log.id is #{log.id.inspect} in :logs_append! job_id: #{data[:id]}" if log.id.to_i == 0 + Log::Part.create!(log_id: log.id, content: chars, number: number, final: final?) + end + rescue ActiveRecord::ActiveRecordError => e + Travis.logger.warn "[warn] could not save log in :logs_append job_id: #{data[:id]}: #{e.message}" + Travis.logger.warn e.backtrace + end + + def notify + job.notify(:log, _log: chars, number: number, final: final?) + rescue => e + Metriks.meter('travis.logs.update.notify.errors').mark + Travis.logger.error("Error notifying of log update: #{e.message} (from #{e.backtrace.first})") + end + + def log + @log ||= Log.where(job_id: job.id).select(:id).first || create_log + end + + def create_log + Travis.logger.warn "[warn] Had to create a log for job_id: #{job.id}!" + job.create_log! + end + + def job + @job ||= Job::Test.find(data[:id]) + end + + def chars + @chars ||= filter(data[:log]) + end + + def number + data[:number] + end + + def final? + !!data[:final] || chars.include?(FINAL) + end + + def data + @data ||= params[:data].symbolize_keys + end + + def filter(chars) + Coder.clean!(chars.to_s.gsub("\0", '')) # postgres seems to have issues with null chars + end + + def measure(name, &block) + Metriks.timer(name).time(&block) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/mailer.rb b/vendor/travis-core/lib/travis/mailer.rb new file mode 100644 index 00000000..8a19cf01 --- /dev/null +++ b/vendor/travis-core/lib/travis/mailer.rb @@ -0,0 +1,26 @@ +require 'action_mailer' +require 'i18n' + +module Travis + module Mailer + class << self + def config + config = Travis.config.smtp + config ? config.to_h : {} + end + + def setup + if config.present? + mailer = ActionMailer::Base + mailer[:delivery_method] = :smtp + mailer[:smtp_settings] = config + @setup = true + end + end + + def setup? + !!@setup + end + end + end +end diff --git a/vendor/travis-core/lib/travis/mailer/user_mailer.rb b/vendor/travis-core/lib/travis/mailer/user_mailer.rb new file mode 100644 index 00000000..3a352648 --- /dev/null +++ b/vendor/travis-core/lib/travis/mailer/user_mailer.rb @@ -0,0 +1,18 @@ +require 'action_mailer' + +class UserMailer < ActionMailer::Base + ActionMailer::Base.append_view_path("#{File.dirname(__FILE__)}/views") + + layout 'contact_email' + + def welcome_email(user) + @user = user + mail(subject: "Welcome to Travis CI!", from: from, to: user.email) do |format| + format.html + end + end + + def from + Travis.config.email.from + end +end diff --git a/vendor/travis-core/lib/travis/mailer/views/layouts/contact_email.html.erb b/vendor/travis-core/lib/travis/mailer/views/layouts/contact_email.html.erb new file mode 100644 index 00000000..ceef3ba3 --- /dev/null +++ b/vendor/travis-core/lib/travis/mailer/views/layouts/contact_email.html.erb @@ -0,0 +1,170 @@ + + + + + + +
+ + +
+ +
+ <%= yield %> +
+ +
+
    +
  • Travis CI GmbH
    • +
  • Rigaerstrasse 8
    • +
  • 10247 Berlin
    • +
  • Germany
    • +
+ + +
+ + diff --git a/vendor/travis-core/lib/travis/mailer/views/user_mailer/welcome_email.html.erb b/vendor/travis-core/lib/travis/mailer/views/user_mailer/welcome_email.html.erb new file mode 100644 index 00000000..3f210a32 --- /dev/null +++ b/vendor/travis-core/lib/travis/mailer/views/user_mailer/welcome_email.html.erb @@ -0,0 +1,54 @@ +
+

Welcome to Travis CI!

+ +

+ Hey <%= @user.name.blank? ? @user.login : @user.name %>, +

+ +

+ We'd like to extend a warm welcome to you and provide with some links to help you get started. +

+ +

+ If you haven't set up your first project yet, head to your + accounts page and enable the project you'd like to test on Travis CI. Push some code, and your repository + will appear on <%= Travis.config.host -%>. +

+ +

+ Are you part of an organization that's already running builds on Travis CI? Great news, we've just finished + synchronizing your permissions from GitHub. You can see the projects you have access to and that have already + been built on Travis CI at <%= Travis.config.host %> so you can + dive in right away. +

+ +

+ Remember to add a .travis.yml file to your project to tell us what steps we should execute to set up your build + environment and run your build. We have sensible defaults, but you're free to customize everything to your + liking. +

+ +

+ You can find all details on how to setup specific languages, the + available configuration options for your + builds, and our build environment in our documentation. Don't forget to setup notifications if + you'd like to be kept up-to-date about your builds on Campfire, HipChat, IRC, and others. +

+ +

+ If you have any questions or issues, shoot us an email. +

+ +

+ Have an awesome day! +

+ +

+ Cheers, +

+ +

+ The Travis CI Team +

+
diff --git a/vendor/travis-core/lib/travis/model.rb b/vendor/travis-core/lib/travis/model.rb new file mode 100644 index 00000000..5c0aa565 --- /dev/null +++ b/vendor/travis-core/lib/travis/model.rb @@ -0,0 +1,60 @@ +# This module is required for preloading classes on JRuby, see +# https://github.com/travis-ci/travis-support/blob/master/lib/core_ext/module/load_constants.rb +# which is used in +# https://github.com/travis-ci/travis-hub/blob/master/lib/travis/hub/cli.rb#L15 +require 'active_record' +require 'core_ext/active_record/base' + +module Travis + class Model < ActiveRecord::Base + require 'travis/model/logs_model' + require 'travis/model/account' + require 'travis/model/annotation' + require 'travis/model/annotation_provider' + require 'travis/model/branch' + require 'travis/model/broadcast' + require 'travis/model/build' + require 'travis/model/commit' + require 'travis/model/email' + require 'travis/model/env_helpers' + require 'travis/model/job' + require 'travis/model/log' + require 'travis/model/membership' + require 'travis/model/organization' + require 'travis/model/permission' + require 'travis/model/repository' + require 'travis/model/request' + require 'travis/model/ssl_key' + require 'travis/model/token' + require 'travis/model/user' + require 'travis/model/url' + + self.abstract_class = true + + cattr_accessor :follower_connection_handler + + class << self + def connection_handler + if Thread.current['Travis.with_follower_connection_handler'] + follower_connection_handler + else + super + end + end + + def establish_follower_connection(spec) + self.follower_connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new unless self.follower_connection_handler + using_follower do + self.establish_connection(spec) + end + end + + def using_follower + Thread.current['Travis.with_follower_connection_handler'] = true + yield + ensure + Thread.current['Travis.with_follower_connection_handler'] = false + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/account.rb b/vendor/travis-core/lib/travis/model/account.rb new file mode 100644 index 00000000..ae471ea4 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/account.rb @@ -0,0 +1,22 @@ +class Account + class << self + def from(record, attrs = {}) + new(record.attributes.merge(:type => record.class.name).merge(attrs)) + end + end + + ATTR_NAMES = [:id, :type, :name, :login, :repos_count, :avatar_url] + + attr_accessor *ATTR_NAMES + + def initialize(attrs) + attrs = attrs.symbolize_keys + ATTR_NAMES.each do |name| + self.send(:"#{name}=", attrs[name]) + end + end + + def ==(other) + id == other.id && type == other.type + end +end diff --git a/vendor/travis-core/lib/travis/model/annotation.rb b/vendor/travis-core/lib/travis/model/annotation.rb new file mode 100644 index 00000000..7a5ee4fc --- /dev/null +++ b/vendor/travis-core/lib/travis/model/annotation.rb @@ -0,0 +1,28 @@ +require "active_record" +require "addressable/uri" +require 'travis/event' + +class Annotation < ActiveRecord::Base + include Travis::Event + + belongs_to :job + belongs_to :annotation_provider + + attr_accessible :description, :url, :job_id, :status + + validates :job_id, presence: true + validates :description, presence: true + validate :validate_url_scheme + + private + def validate_url_scheme + return unless self.url + + uri = Addressable::URI.parse(self.url) + unless %w[http https].include?(uri.scheme) + errors.add(:url, 'URL must use http or https scheme') + end + rescue Addressable::URI::InvalidURIError + errors.add(:url, 'URL is invalid') + end +end diff --git a/vendor/travis-core/lib/travis/model/annotation_provider.rb b/vendor/travis-core/lib/travis/model/annotation_provider.rb new file mode 100644 index 00000000..c5193662 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/annotation_provider.rb @@ -0,0 +1,20 @@ +require 'active_record' +require 'travis/model/encrypted_column' + +class AnnotationProvider < ActiveRecord::Base + has_many :annotations + + serialize :api_key, Travis::Model::EncryptedColumn.new + + def self.authenticate_provider(username, key) + provider = where(api_username: username).first + + return unless provider && provider.api_key == key + + provider + end + + def annotation_for_job(job_id) + annotations.where(job_id: job_id).first || annotations.build(job_id: job_id) + end +end diff --git a/vendor/travis-core/lib/travis/model/branch.rb b/vendor/travis-core/lib/travis/model/branch.rb new file mode 100644 index 00000000..ad8a21d6 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/branch.rb @@ -0,0 +1,6 @@ +require 'travis/model' + +class Branch < Travis::Model + belongs_to :repository + belongs_to :last_build, class_name: 'Build' +end diff --git a/vendor/travis-core/lib/travis/model/broadcast.rb b/vendor/travis-core/lib/travis/model/broadcast.rb new file mode 100644 index 00000000..cf5214f0 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/broadcast.rb @@ -0,0 +1,30 @@ +require 'travis/model' + +class Broadcast < Travis::Model + belongs_to :recipient, polymorphic: true + + class << self + def by_user(user) + sql = %( + recipient_type IS NULL OR + recipient_type = ? AND recipient_id IN(?) OR + recipient_type = ? AND recipient_id = ? OR + recipient_type = ? AND recipient_id IN (?) + ) + active.where(sql, 'Organization', user.organization_ids, 'User', user.id, 'Repository', user.repository_ids) + end + + def by_repo(repository) + sql = %( + recipient_type IS NULL OR + recipient_type = ? AND recipient_id = ? OR + recipient_type = ? AND recipient_id = ? + ) + active.where(sql, 'Repository', repository.id, repository.owner_type, repository.owner_id) + end + + def active + where('created_at >= ? AND (expired IS NULL OR expired <> ?)', 14.days.ago, true).order('id DESC') + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build.rb b/vendor/travis-core/lib/travis/model/build.rb new file mode 100644 index 00000000..636648af --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build.rb @@ -0,0 +1,228 @@ +require 'core_ext/hash/deep_symbolize_keys' +require 'simple_states' +require 'travis/model' +require 'travis/services/next_build_number' + +# Build currently models a central but rather abstract domain entity: the thing +# that is triggered by a Github request (service hook ping). +# +# Build groups a matrix of Job::Test instances, and belongs to a Request (and +# thus Commit as well as a Repository). +# +# A Build is created when its Request was configured (by fetching .travis.yml) +# and approved (e.g. not excluded by the configuration). Once a Build is +# created it will expand its matrix according to the given configuration and +# create the according Job::Test instances. Each Job::Test instance will +# trigger a test run remotely (on the worker). Once all Job::Test instances +# have finished the Build will be finished as well. +# +# Each of these state changes (build:created, job:started, job:finished, ...) +# will issue events that are listened for by the event handlers contained in +# travis/notification. These event handlers then send out various notifications +# of various types through email, pusher and irc, archive builds and queue +# jobs for the workers. +# +# Build is split up to several modules: +# +# * Build - ActiveRecord structure, validations and scopes +# * States - state definitions and events +# * Denormalize - some state changes denormalize attributes to the build's +# repository (e.g. Build#started_at gets propagated to +# Repository#last_started_at) +# * Matrix - logic related to expanding the build matrix, normalizing +# configuration for Job::Test instances, evaluating the +# final build result etc. +# * Messages - helpers for evaluating human readable result messages +# (e.g. "Still Failing") +# * Events - helpers that are used by notification handlers (and that +# TODO probably should be cleaned up and moved to +# travis/notification) +class Build < Travis::Model + require 'travis/model/build/config' + require 'travis/model/build/denormalize' + require 'travis/model/build/update_branch' + require 'travis/model/build/matrix' + require 'travis/model/build/metrics' + require 'travis/model/build/result_message' + require 'travis/model/build/states' + require 'travis/model/env_helpers' + + include Matrix, States, SimpleStates + + belongs_to :commit + belongs_to :request + belongs_to :repository, autosave: true + belongs_to :owner, polymorphic: true + has_many :matrix, as: :source, order: :id, class_name: 'Job::Test', dependent: :destroy + has_many :events, as: :source + + validates :repository_id, :commit_id, :request_id, presence: true + + serialize :config + + delegate :same_repo_pull_request?, :to => :request + + class << self + def recent + where(state: ['failed', 'passed']).order('id DESC').limit(25) + end + + def running + where(state: ['started']).order('started_at DESC') + end + + def was_started + where('state <> ?', :created) + end + + def finished + where(state: [:finished, :passed, :failed, :errored, :canceled]) # TODO extract + end + + def on_state(state) + where(state.present? ? ['builds.state IN (?)', state] : []) + end + + def on_branch(branch) + api_and_pushes.where(branch.present? ? ['branch IN (?)', normalize_to_array(branch)] : []) + end + + def by_event_type(event_types) + event_types = Array(event_types).flatten + event_types << 'push' if event_types.empty? + where(event_type: event_types) + end + + def pushes + where(event_type: 'push') + end + + def pull_requests + where(event_type: 'pull_request') + end + + def api_and_pushes + by_event_type(['api', 'push']) + end + + def previous(build) + where('builds.repository_id = ? AND builds.id < ?', build.repository_id, build.id).finished.descending.limit(1).first + end + + def descending + order(arel_table[:id].desc) + end + + def paged(options) + page = (options[:page] || 1).to_i + limit(per_page).offset(per_page * (page - 1)) + end + + def last_build_on(options) + scope = descending + scope = scope.on_state(options[:state]) if options[:state] + scope = scope.on_branch(options[:branch]) if options[:branch] + scope.first + end + + def last_state_on(options) + last_build_on(options).try(:state).try(:to_sym) + end + + def older_than(build = nil) + scope = order('number::integer DESC').paged({}) # TODO in which case we'd call older_than without an argument? + scope = scope.where('number::integer < ?', (build.is_a?(Build) ? build.number : build).to_i) if build + scope + end + + protected + + def normalize_to_array(object) + Array(object).compact.join(',').split(',') + end + + def per_page + 25 + end + end + + # set the build number and expand the matrix; downcase language + before_create do + next_build_number = Travis::Services::NextBuildNumber.new(repository_id: repository.id).run + self.number = next_build_number + self.previous_state = last_finished_state_on_branch + self.event_type = request.event_type + self.pull_request_title = request.pull_request_title + self.pull_request_number = request.pull_request_number + self.branch = commit.branch + expand_matrix + end + + after_create do + UpdateBranch.new(self).update_last_build unless pull_request? + end + + after_save do + unless cached_matrix_ids + update_column(:cached_matrix_ids, to_postgres_array(matrix_ids)) + end + end + + # AR 3.2 does not handle pg arrays and the plugins supporting them + # do not work well with jdbc drivers + # TODO: remove this once we're on >= 4.0 + def cached_matrix_ids + if (value = super) && value =~ /^{/ + value.gsub(/^{|}$/, '').split(',').map(&:to_i) + end + end + + def matrix_ids + matrix.map(&:id) + end + + def secure_env_enabled? + !pull_request? || same_repo_pull_request? + end + alias addons_enabled? secure_env_enabled? + + def config=(config) + super((config || {}).deep_symbolize_keys) + end + + def config + @config ||= Config.new(super, multi_os: repository.multi_os_enabled?).normalize + end + + def obfuscated_config + Config.new(config, key_fetcher: lambda { self.repository.key }).obfuscate + end + + def cancelable? + matrix.any? { |job| job.cancelable? } + end + + def pull_request? + event_type == 'pull_request' + end + + # COMPAT: used in http api v1, deprecate as soon as v1 gets retired + def result + state.try(:to_sym) == :passed ? 0 : 1 + end + + def on_default_branch? + branch == repository.default_branch + end + + private + + def last_finished_state_on_branch + repository.builds.finished.last_state_on(branch: commit.branch) + end + + def to_postgres_array(ids) + ids = ids.compact.uniq + "{#{ids.map { |id| id.to_i.to_s }.join(',')}}" unless ids.empty? + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config.rb b/vendor/travis-core/lib/travis/model/build/config.rb new file mode 100644 index 00000000..80ac807c --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config.rb @@ -0,0 +1,119 @@ +require 'travis/model/build/config/dist' +require 'travis/model/build/config/env' +require 'travis/model/build/config/features' +require 'travis/model/build/config/group' +require 'travis/model/build/config/language' +require 'travis/model/build/config/matrix' +require 'travis/model/build/config/obfuscate' +require 'travis/model/build/config/os' +require 'travis/model/build/config/yaml' + +class Build + class Config + NORMALIZERS = [Features, Yaml, Env, Language, Group, Dist] + + DEFAULT_LANG = 'ruby' + + ENV_KEYS = [ + :compiler, + :crystal, + :csharp, + :d, + :dart, + :elixir, + :env, + :fsharp, + :gemfile, + :ghc, + :go, + :haxe, + :jdk, + :julia, + :mono, + :node_js, + :otp_release, + :perl, + :perl6, + :php, + :python, + :ruby, + :rust, + :rvm, + :r, + :scala, + :smalltalk, + :smalltalk_config, + :visualbasic, + :xcode_scheme, + :xcode_sdk + ] + + EXPANSION_KEYS_FEATURE = [:os] + + EXPANSION_KEYS_LANGUAGE = { + 'c' => [:compiler], + 'c++' => [:compiler], + 'clojure' => [:lein, :jdk], + 'cpp' => [:compiler], + 'crystal' => [:crystal], + 'csharp' => [:csharp, :mono], + 'd' => [:d], + 'dart' => [:dart], + 'elixir' => [:elixir, :otp_release], + 'erlang' => [:otp_release], + 'fsharp' => [:fsharp, :mono], + 'go' => [:go], + 'groovy' => [:jdk], + 'haskell' => [:ghc], + 'haxe' => [:haxe], + 'java' => [:jdk], + 'julia' => [:julia], + 'node_js' => [:node_js], + 'objective-c' => [:rvm, :gemfile, :xcode_sdk, :xcode_scheme], + 'perl' => [:perl], + 'perl6' => [:perl6], + 'php' => [:php], + 'python' => [:python], + 'ruby' => [:rvm, :gemfile, :jdk, :ruby], + 'rust' => [:rust], + 'r' => [:r], + 'scala' => [:scala, :jdk], + 'smalltalk' => [:smalltalk, :smalltalk_config], + 'visualbasic' => [:visualbasic, :mono] + } + + EXPANSION_KEYS_UNIVERSAL = [:env, :branch] + + def self.matrix_keys_for(config, options = {}) + keys = matrix_keys(config, options) + keys & config.keys.map(&:to_sym) + end + + def self.matrix_keys(config, options = {}) + lang = Array(config.symbolize_keys[:language]).first + keys = ENV_KEYS + keys &= EXPANSION_KEYS_LANGUAGE.fetch(lang, EXPANSION_KEYS_LANGUAGE[DEFAULT_LANG]) + keys << :os if options[:multi_os] + keys += [:dist, :group] if options[:dist_group_expansion] + keys | EXPANSION_KEYS_UNIVERSAL + end + + attr_reader :config, :options + + def initialize(config, options = {}) + @config = (config || {}).deep_symbolize_keys + @options = options + end + + def normalize + normalizers = options[:multi_os] ? NORMALIZERS : NORMALIZERS + [OS] + normalizers.inject(config) do |config, normalizer| + normalizer.new(config, options).run + end + end + + def obfuscate + Obfuscate.new(config, options).run + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/dist.rb b/vendor/travis-core/lib/travis/model/build/config/dist.rb new file mode 100644 index 00000000..0b4d87f6 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/dist.rb @@ -0,0 +1,59 @@ +class Build + class Config + class Dist + DIST_LANGUAGE_MAP = { + 'objective-c' => 'osx' + }.freeze + DIST_OS_MAP = { + 'osx' => 'osx' + }.freeze + DIST_SERVICES_MAP = { + 'docker' => 'trusty' + }.freeze + DEFAULT_DIST = 'precise' + + attr_reader :config, :options + + def initialize(config, options) + @config = config + @options = options + end + + def run + return config unless config_hashy? + return config if config.key?(:dist) || config.key?('dist') + + config.dup.tap do |c| + c.merge!(dist: dist_for_config) + + matrix = c.fetch(:matrix, {}) + return c unless config_hashy?(matrix) + + included = matrix.fetch(:include, []) || [] + + included.each do |inc| + next unless config_hashy?(inc) + next if inc.key?(:dist) || inc.key?('dist') + inc.merge!(dist: dist_for_config(inc)) + end + end + end + + private + + def config_hashy?(h = config) + %w(key? dup merge! fetch).all? { |m| h.respond_to?(m) } + end + + def dist_for_config(h = config) + return DIST_LANGUAGE_MAP[h[:language]] if + DIST_LANGUAGE_MAP.key?(h[:language]) + (Array(h[:services]) || []).each do |service| + return DIST_SERVICES_MAP[service] if DIST_SERVICES_MAP.key?(service) + end + return DEFAULT_DIST if options[:multi_os] + DIST_OS_MAP.fetch(Array(h[:os]).first, DEFAULT_DIST) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/env.rb b/vendor/travis-core/lib/travis/model/build/config/env.rb new file mode 100644 index 00000000..ba4c02e6 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/env.rb @@ -0,0 +1,50 @@ +require 'core_ext/hash/compact' + +class Build + class Config + class Env < Struct.new(:config, :options) + def run + case config[:env] + when Hash + config.merge(normalize_hash(config[:env])).compact + when Array + config.merge(env: config[:env].map { |value| normalize_value(value) }) + else + config + end + end + + def normalize_hash(env) + if env[:global] || env[:matrix] + { global_env: normalize_values(env[:global]), env: normalize_values(env[:matrix]) } + else + { env: normalize_values(env) } + end + end + + def normalize_values(values) + values = [values].compact unless values.is_a?(Array) + values.map { |value| normalize_value(value) } unless values.empty? + end + + def normalize_value(value) + case value + when Hash + to_env_var(value) + when Array + value.map { |value| to_env_var(value) } + else + value + end + end + + def to_env_var(hash) + if hash.is_a?(Hash) && !hash.key?(:secure) + hash.map { |name, value| "#{name}=#{value}" }.join(' ') + else + hash + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/features.rb b/vendor/travis-core/lib/travis/model/build/config/features.rb new file mode 100644 index 00000000..5c08ca8f --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/features.rb @@ -0,0 +1,22 @@ +require 'core_ext/hash/compact' + +class Build + class Config + class Features < Struct.new(:config, :options) + def run + config = self.config + config = remove_multi_os(config) unless options[:multi_os] + config + end + + def remove_multi_os(config) + config.delete(:os) + includes = config[:matrix].is_a?(Hash) && config[:matrix][:include] + return config unless includes.is_a?(Array) + includes = includes.each { |c| c.delete(:os) if c.is_a?(Hash) }.uniq + config[:matrix][:include] = includes + config + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/group.rb b/vendor/travis-core/lib/travis/model/build/config/group.rb new file mode 100644 index 00000000..bac46c36 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/group.rb @@ -0,0 +1,18 @@ +class Build + class Config + class Group + DEFAULT_GROUP = 'stable' + + attr_reader :config + + def initialize(config, *) + @config = config + end + + def run + return config if config.key?(:group) || config.key?('group') + config.merge(group: DEFAULT_GROUP) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/language.rb b/vendor/travis-core/lib/travis/model/build/config/language.rb new file mode 100644 index 00000000..b6108de3 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/language.rb @@ -0,0 +1,27 @@ +require 'active_support/core_ext/array/wrap' + +class Build + class Config + class Language < Struct.new(:config, :options) + def run + config[:language] = Array.wrap(config[:language]).first.to_s.downcase + config[:language] = DEFAULT_LANG if config[:language].empty? + config.select { |key, _| include_key?(key) } + end + + private + + def include_key?(key) + matrix_keys.include?(key) || !known_env_key?(key) + end + + def matrix_keys + Config.matrix_keys(config, options) + end + + def known_env_key?(key) + (ENV_KEYS | EXPANSION_KEYS_FEATURE).include?(key) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/matrix.rb b/vendor/travis-core/lib/travis/model/build/config/matrix.rb new file mode 100644 index 00000000..8f5bcd3b --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/matrix.rb @@ -0,0 +1,108 @@ +require 'active_support/core_ext/hash/except' +require 'active_support/core_ext/array/wrap' + +class Build + class Config + class Matrix + attr_reader :config, :options + + def initialize(config, options = {}) + @config = config || {} + @options = options + end + + def expand + configs = expand_matrix + configs = include_matrix_configs(exclude_matrix_configs(configs)) + configs = configs.map { |config| cleanup_config(merge_config(Hash[config])) } + configs.map { |config| Build::Config::OS.new(config, options).run } + end + + def allow_failure_configs + Array(settings[:allow_failures] || []).select do |config| + # TODO check with @drogus how/when this might happen + config = config.to_hash.symbolize_keys if config.respond_to?(:to_hash) + end + end + + def fast_finish? + settings[:fast_finish] + end + + private + + def settings + @_settings ||= config[:matrix] || {} + @_settings = {} if @_settings.is_a?(Array) + @_settings + end + + def expand_matrix + rows = config.slice(*expand_keys).values.select { |value| value.is_a?(Array) } + max_size = rows.max_by(&:size).try(:size) || 1 + + array = expand_keys.inject([]) do |result, key| + values = Array.wrap(config[key]) + values += [values.last] * (max_size - values.size) + result << values.map { |value| [key, value] } + end + + permutations(array).uniq + end + + # recursively builds up permutations of values in the rows of a nested array + def permutations(base, result = []) + base = base.dup + base.empty? ? [result] : base.shift.map { |value| permutations(base, result + [value]) }.flatten(1) + end + + def expand_keys + @expand_keys ||= config.keys.map(&:to_sym) & Config.matrix_keys_for(config, options) + end + + def exclude_matrix_configs(configs) + configs.reject { |config| exclude_config?(config) } + end + + def exclude_config?(config) + exclude_configs = normalize_matrix_filter_configs(settings[:exclude] || []) + config = config.map { |config| [config[0].to_s, *config[1..-1]] }.sort + exclude_configs.any? do |excluded| + excluded.all? { |matrix_key| config.include? matrix_key } + end + end + + def include_matrix_configs(configs) + include_configs = normalize_matrix_filter_configs(settings[:include] || []) + if configs.flatten.empty? && settings.has_key?(:include) + include_configs + else + configs + include_configs + end + end + + def normalize_matrix_filter_configs(configs) + configs = configs.select { |c| c.is_a?(Hash) } + configs = configs.compact.map(&:stringify_keys) + configs.map(&:to_a).map(&:sort) + end + + def merge_config(row) + config.select { |key, value| include_key?(key) }.merge(row) + end + + def cleanup_config(config) + config.delete(:matrix) + config + end + + def include_key?(key) + Config.matrix_keys_for(config, options).include?(key.to_sym) || !known_env_key?(key.to_sym) + end + + def known_env_key?(key) + (ENV_KEYS | EXPANSION_KEYS_FEATURE).include?(key) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/obfuscate.rb b/vendor/travis-core/lib/travis/model/build/config/obfuscate.rb new file mode 100644 index 00000000..b1f8a39e --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/obfuscate.rb @@ -0,0 +1,53 @@ +require 'active_support/core_ext/hash/except' +require 'active_support/core_ext/array/wrap' +require 'travis/secure_config' + +class Build + class Config + class Obfuscate < Struct.new(:config, :options) + ENV_VAR_PATTERN = /(?<=\=)(?:(?['"]).*?[^\\]\k|(.*?)(?= \w+=|$))/ + + def run + config = self.config.except(:source_key) + config[:env] = obfuscate(config[:env]) if config[:env] + config + end + + private + + def obfuscate(env) + Array.wrap(env).map do |value| + obfuscate_values(value).join(' ') + end + end + + def obfuscate_values(values) + Array.wrap(values).compact.map do |value| + obfuscate_value(value) + end + end + + def obfuscate_value(value) + secure.decrypt(value) do |decrypted| + obfuscate_env_vars(decrypted) + end + end + + def obfuscate_env_vars(line) + if line.respond_to?(:gsub) + line.gsub(ENV_VAR_PATTERN) { |val| '[secure]' } + else + '[One of the secure variables in your .travis.yml has an invalid format.]' + end + end + + def secure + @secure ||= Travis::SecureConfig.new(key) + end + + def key + @key ||= options[:key_fetcher].call + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/os.rb b/vendor/travis-core/lib/travis/model/build/config/os.rb new file mode 100644 index 00000000..e37fde19 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/os.rb @@ -0,0 +1,27 @@ +class Build + class Config + class OS + OS_LANGUAGE_MAP = { + 'objective-c' => 'osx', + } + DEFAULT_OS = 'linux' + + attr_reader :config + + def initialize(config, _) + @config = config + end + + def run + return config if config.key?(:os) || config.key?('os') + config.merge(os: os_for_language) + end + + private + + def os_for_language + OS_LANGUAGE_MAP.fetch(config[:language], DEFAULT_OS) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/config/yaml.rb b/vendor/travis-core/lib/travis/model/build/config/yaml.rb new file mode 100644 index 00000000..1afe86ff --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/config/yaml.rb @@ -0,0 +1,14 @@ +class Build + class Config + class Yaml < Struct.new(:config, :options) + def run + normalize(config) + end + + def normalize(hash) + Hash[hash.map { |key, value| [key == true ? :on : key, value] }] + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/model/build/denormalize.rb b/vendor/travis-core/lib/travis/model/build/denormalize.rb new file mode 100644 index 00000000..d01a5127 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/denormalize.rb @@ -0,0 +1,34 @@ +class Build + + # The build's start and finish events (state changes) trigger denormalization + # of certain attributes to the repository in order to disburden the db a bit. + # + # E.g. on `start` the `started_at` attribute of a build gets set to its + # repository's `last_started_at` attribute. Likewise on `finish` the + # `finished_at` and `state` attributes are set to `last_build_finished_at` and + # `last_build_state` on the repository. + # + # These attributes are used in the repositories list and thus read frequently. + module Denormalize + def denormalize(event, *args) + repository.update_attributes!(denormalize_attributes_for(event)) if denormalize?(event) + end + + DENORMALIZE = { + start: %w(id number state duration started_at finished_at), + finish: %w(state duration finished_at), + reset: %w(state duration started_at finished_at), + cancel: %w(state duration finished_at) + } + + def denormalize?(event) + DENORMALIZE.key?(event) + end + + def denormalize_attributes_for(event) + DENORMALIZE[event].inject({}) do |result, key| + result.merge(:"last_build_#{key}" => send(key)) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/matrix.rb b/vendor/travis-core/lib/travis/model/build/matrix.rb new file mode 100644 index 00000000..25bf7b67 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/matrix.rb @@ -0,0 +1,114 @@ +require 'active_support/concern' +require 'active_support/core_ext/hash/keys' +require 'core_ext/hash/deep_symbolize_keys' + +class Build + + # A Build contains a number of Job::Test instances that make up the build + # matrix. + # + # The matrix is defined in the build configuration (`.travis.yml`) and + # expanded (evaluated and instantiated) when the Build is created. + # + # A build matrix has 1 to 3 dimensions and can be defined by specifying + # multiple values for either of: + # + # * a language/vm variant (e.g. 1.9.2, rbx, jruby for a Ruby build) + # * a dependency definition (e.g. a Gemfile for a Ruby build) + # * an arbitrary env key that can be used from within the test suite in + # order to branch out specific variations of the test run + module Matrix + extend ActiveSupport::Concern + + def matrix_finished? + if matrix_config.fast_finish? + required_jobs.all?(&:finished?) || required_jobs.any?(&:finished_unsuccessfully?) + else + matrix.all?(&:finished?) + end + end + + def matrix_duration + matrix_finished? ? matrix.inject(0) { |duration, job| duration + job.duration.to_i } : nil + end + + def matrix_state + if required_jobs.blank? + :passed + elsif required_jobs.any?(&:canceled?) + :canceled + elsif required_jobs.any?(&:errored?) + :errored + elsif required_jobs.any?(&:failed?) + :failed + elsif required_jobs.all?(&:passed?) + :passed + else + raise InvalidMatrixStateException.new(matrix) + end + end + + # expand the matrix (i.e. create test jobs) and update the config for each job + def expand_matrix + matrix_config.expand.each_with_index do |row, ix| + attributes = self.attributes.slice(*Job.column_names - ['status', 'result']).symbolize_keys + attributes.merge!( + owner: owner, + number: "#{number}.#{ix + 1}", + config: row, + log: Log.new + ) + matrix.build(attributes) + end + matrix_allow_failures # TODO should be able to join this with the loop above + matrix + end + + def expand_matrix! + expand_matrix + save! + end + + # Return only the child builds whose config matches against as passed hash + # e.g. build.filter_matrix(rvm: '1.8.7', env: 'DB=postgresql') + def filter_matrix(config) + config.blank? ? matrix : matrix.select { |job| job.matches_config?(config) } + end + + private + + def matrix_config + @matrix_config ||= Config::Matrix.new(config, multi_os: repository.multi_os_enabled?, dist_group_expansion: repository.dist_group_expansion_enabled?) + end + + def matrix_allow_failures + configs = matrix_config.allow_failure_configs + jobs = configs.map { |config| filter_matrix(config) }.flatten + jobs.each { |job| job.allow_failure = true } + end + + def required_jobs + @required_jobs ||= matrix.reject { |test| test.allow_failure? } + end + end + + class InvalidMatrixStateException < StandardError + attr_reader :matrix + + def initialize(matrix) + @matrix = matrix + end + + def to_s + sanitized = matrix.map do |job| + "\n\tid: #{job.id}, repository: #{job.repository.slug}, state: #{job.state}, " + + "allow_failure: #{job.allow_failure}, " + + "created_at: #{job.created_at.inspect}, queued_at: #{job.queued_at.inspect}, " + + "started_at: #{job.started_at.inspect}, finished_at: #{job.finished_at.inspect}, " + + "canceled_at: #{job.canceled_at.inspect}, updated_at: #{job.updated_at.inspect}" + end.join + + "Invalid build matrix state detected.\nMatrix: #{sanitized}" + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/metrics.rb b/vendor/travis-core/lib/travis/model/build/metrics.rb new file mode 100644 index 00000000..897e05ef --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/metrics.rb @@ -0,0 +1,15 @@ +require 'travis/model/build' +class Build + module Metrics + def start(data = {}) + super + meter 'travis.builds.start.delay', started_at - request.created_at + end + + private + + def meter(name, time) + Metriks.timer(name).update(time) + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/result_message.rb b/vendor/travis-core/lib/travis/model/build/result_message.rb new file mode 100644 index 00000000..669c0492 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/result_message.rb @@ -0,0 +1,79 @@ +require 'hashr' + +class Build + class ResultMessage + + # TODO extract to I18n + SHORT = { + pending: 'Pending', + passed: 'Passed', + failed: 'Failed', + broken: 'Broken', + fixed: 'Fixed', + failing: 'Still Failing', + errored: 'Errored', + canceled: 'Canceled' + } + + FULL = { + pending: 'The build is pending.', + passed: 'The build passed.', + failed: 'The build failed.', + broken: 'The build was broken.', + fixed: 'The build was fixed.', + failing: 'The build is still failing.', + errored: 'The build has errored.', + canceled: 'The build was canceled.' + } + + EMAIL = { + pending: 'Build #%d is pending.', + passed: 'Build #%d passed.', + failed: 'Build #%d failed.', + broken: 'Build #%d was broken.', + fixed: 'Build #%d was fixed.', + failing: 'Build #%d is still failing.', + errored: 'Build #%d has errored.', + canceled: 'Build #%d was canceled.' + } + + + attr_reader :build + + def initialize(build) + build = Hashr.new(build) if build.is_a?(Hash) + @build = build + end + + def short + SHORT[result_key] + end + + def full + FULL[result_key] + end + + def email + EMAIL[result_key] % build.number.to_i + end + + private + + def result_key + current = build.state.try(:to_sym) + previous = build.previous_state.try(:to_sym) + + if [:created, :queued, :received, :started].include?(current) + :pending + elsif previous == :passed && current == :failed + :broken + elsif previous == :failed && current == :passed + :fixed + elsif previous == :failed && current == :failed + :failing + else + current + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/states.rb b/vendor/travis-core/lib/travis/model/build/states.rb new file mode 100644 index 00000000..25b1193f --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/states.rb @@ -0,0 +1,109 @@ +require 'active_support/concern' +require 'simple_states' + +class Build + + # A Build goes through the following lifecycle: + # + # * A newly created Build is in the `created` state. + # * When started it sets its `started_at` attribute from the given + # (worker) payload. + # * A build won't be restarted if it already is started (each matrix job + # will try to start it). + # * A build will be finished only if all matrix jobs are finished (each + # matrix job will try to finish it). + # * After both `start` and `finish` events the build will denormalize + # attributes to its repository and notify event listeners. + module States + extend ActiveSupport::Concern + include Denormalize, Travis::Event + + included do + include SimpleStates + + states :created, :received, :started, :passed, :failed, :errored, :canceled + + event :receive, to: :received, unless: [:received?, :started?, :failed?, :errored?] + event :start, to: :started, unless: [:started?, :failed?, :errored?] + event :finish, to: :finished, if: :should_finish? + event :reset, to: :created + event :cancel, to: :canceled, if: :cancelable? + event :all, after: [:denormalize, :notify] + end + + def should_finish? + matrix_finished? && !finished? + end + + def receive(data = {}) + self.received_at = data[:received_at] + end + + def start(data = {}) + self.started_at = data[:started_at] + end + + def finish(data = {}) + self.state = matrix_state + self.duration = matrix_duration + self.finished_at = data[:finished_at] + + save! + end + + def cancel(options = {}) + matrix.each do |job| + job.cancel! + end + + finalize_cancel + end + + def finalize_cancel + self.state = matrix_state + self.duration = matrix_duration + self.canceled_at = Time.now + self.finished_at = Time.now + + save! + end + + def cancel_job + if matrix_finished? + finalize_cancel + denormalize(:cancel) + end + end + + def reset(options = {}) + self.state = :created unless matrix.any? { |job| job.state == :started } + %w(duration started_at finished_at).each { |attr| write_attribute(attr, nil) } + matrix.each(&:reset!) if options[:reset_matrix] + end + + def resetable? + finished? && !invalid_config? + end + + def invalid_config? + config[:".result"] == "parse_error" + end + + def pending? + created? || started? + end + + def finished? + passed? || failed? || errored? || canceled? + end + + def color + pending? ? 'yellow' : passed? ? 'green' : 'red' + end + + def notify(event, *args) + event = :create if event == :reset + super + end + end +end diff --git a/vendor/travis-core/lib/travis/model/build/update_branch.rb b/vendor/travis-core/lib/travis/model/build/update_branch.rb new file mode 100644 index 00000000..9ed8e936 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/build/update_branch.rb @@ -0,0 +1,37 @@ +class Build + class UpdateBranch < Struct.new(:build) + MSGS = { + update: 'Setting last_build_id to %s on branch %s, repo %s', + kaputt: 'Inconsistent branch.last_build_id=%s on branch: %s (repo: %s). Should be: %s.' + } + + def update_last_build + logger.info MSGS[:update] % [build.id, branch_name, repository.slug] + branch.update_attributes!(last_build_id: build.id) + validate_branch_last_build_id # TODO double check and remove after a few days + end + + private + + def validate_branch_last_build_id + return if branch.reload.last_build_id == build.id + logger.warn MSGS[:kaputt] % [branch.last_build_id, branch_name, repository.slug, build.id] + end + + def branch + Branch.where(repository_id: repository.id, name: branch_name).first_or_create + end + + def branch_name + build.branch || 'master' + end + + def repository + build.repository + end + + def logger + Travis.logger + end + end +end diff --git a/vendor/travis-core/lib/travis/model/commit.rb b/vendor/travis-core/lib/travis/model/commit.rb new file mode 100644 index 00000000..f21aeb26 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/commit.rb @@ -0,0 +1,28 @@ +require 'travis/model' + +# Encapsulates a commit that a Build belongs to (and that a Github Request +# referred to). +class Commit < Travis::Model + has_one :request + belongs_to :repository + + validates :commit, :branch, :committed_at, :presence => true + + def pull_request? + ref =~ %r(^refs/pull/\d+/merge$) + end + + def pull_request_number + if pull_request? && (num = ref.scan(%r(^refs/pull/(\d+)/merge$)).flatten.first) + num.to_i + end + end + + def range + if pull_request? + "#{request.base_commit}...#{request.head_commit}" + elsif compare_url && compare_url =~ /\/([0-9a-f]+\^*\.\.\.[0-9a-f]+\^*$)/ + $1 + end + end +end diff --git a/vendor/travis-core/lib/travis/model/email.rb b/vendor/travis-core/lib/travis/model/email.rb new file mode 100644 index 00000000..b652893f --- /dev/null +++ b/vendor/travis-core/lib/travis/model/email.rb @@ -0,0 +1,5 @@ +require 'travis/model' + +class Email < Travis::Model + belongs_to :user +end diff --git a/vendor/travis-core/lib/travis/model/encrypted_column.rb b/vendor/travis-core/lib/travis/model/encrypted_column.rb new file mode 100644 index 00000000..67045051 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/encrypted_column.rb @@ -0,0 +1,105 @@ +require 'securerandom' +require 'base64' + +class Travis::Model::EncryptedColumn + attr_reader :disable, :options + alias disabled? disable + + def initialize(options = {}) + @options = options || {} + @disable = self.options[:disable] + @key = self.options[:key] + end + + def enabled? + !disabled? + end + + def load(data) + return nil unless data + + data = data.to_s + + decrypt?(data) ? decrypt(data) : data + end + + def dump(data) + encrypt?(data) ? encrypt(data.to_s) : data + end + + def key + @key || config.key + end + + def iv + SecureRandom.hex(8) + end + + def prefix + '--ENCR--' + end + + def decrypt?(data) + data.present? && (!use_prefix? || prefix_used?(data)) + end + + def encrypt?(data) + data.present? && enabled? + end + + def prefix_used?(data) + data[0..7] == prefix + end + + def decrypt(data) + data = data[8..-1] if prefix_used?(data) + + data = decode data + + iv = data[-16..-1] + data = data[0..-17] + + aes = create_aes :decrypt, key.to_s, iv + + result = aes.update(data) + aes.final + end + + def encrypt(data) + iv = self.iv + + aes = create_aes :encrypt, key.to_s, iv + + encrypted = aes.update(data) + aes.final + + encrypted = "#{encrypted}#{iv}" + encrypted = encode encrypted + encrypted = "#{prefix}#{encrypted}" if use_prefix? + encrypted + end + + def use_prefix? + options.has_key?(:use_prefix) ? options[:use_prefix] : Travis::Features.feature_inactive?(:db_encryption_prefix) + end + + def create_aes(mode = :encrypt, key, iv) + aes = OpenSSL::Cipher::AES.new(256, :CBC) + + aes.send(mode) + aes.key = key + aes.iv = iv + + aes + end + + def config + Travis.config.encryption + end + + def decode(str) + Base64.strict_decode64 str + end + + def encode(str) + Base64.strict_encode64 str + end +end diff --git a/vendor/travis-core/lib/travis/model/env_helpers.rb b/vendor/travis-core/lib/travis/model/env_helpers.rb new file mode 100644 index 00000000..5ba5a6c3 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/env_helpers.rb @@ -0,0 +1,10 @@ +module Travis::Model::EnvHelpers + def obfuscate_env(vars) + vars = [vars] unless vars.is_a?(Array) + vars.compact.map do |var| + repository.key.secure.decrypt(var) do |decrypted| + Travis::Helpers.obfuscate_env_vars(decrypted) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/job.rb b/vendor/travis-core/lib/travis/model/job.rb new file mode 100644 index 00000000..1d86e278 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/job.rb @@ -0,0 +1,235 @@ +require 'travis/model' +require 'active_support/core_ext/hash/deep_dup' +require 'travis/model/build/config/language' + +# Job models a unit of work that is run on a remote worker. +# +# There currently only one job type: +# +# * Job::Test belongs to a Build (one or many Job::Test instances make up a +# build matrix) and executes a test suite with parameters defined in the +# configuration. +class Job < Travis::Model + require 'travis/model/job/queue' + require 'travis/model/job/test' + require 'travis/model/env_helpers' + + WHITELISTED_ADDONS = %w( + apt + apt_packages + apt_sources + firefox + hosts + mariadb + postgresql + ssh_known_hosts + ).freeze + + class << self + # what we return from the json api + def queued(queue = nil) + scope = where(state: [:created, :queued]) + scope = scope.where(queue: queue) if queue + scope + end + + # what needs to be queued up + def queueable(queue = nil) + scope = where(state: :created).order('jobs.id') + scope = scope.where(queue: queue) if queue + scope + end + + # what already is queued or started + def running(queue = nil) + scope = where(state: [:queued, :received, :started]).order('jobs.id') + scope = scope.where(queue: queue) if queue + scope + end + + def unfinished + # TODO conflate Job and Job::Test and use States::FINISHED_STATES + where('state NOT IN (?)', [:finished, :passed, :failed, :errored, :canceled]) + end + + def owned_by(owner) + where(owner_id: owner.id, owner_type: owner.class.to_s) + end + end + + include Travis::Model::EnvHelpers + + has_one :log, dependent: :destroy + has_many :events, as: :source + has_many :annotations, dependent: :destroy + + belongs_to :repository + belongs_to :commit + belongs_to :source, polymorphic: true, autosave: true + belongs_to :owner, polymorphic: true + + validates :repository_id, :commit_id, :source_id, :source_type, :owner_id, :owner_type, presence: true + + serialize :config + + delegate :request_id, to: :source # TODO denormalize + delegate :pull_request?, to: :commit + delegate :secure_env_enabled?, :addons_enabled?, to: :source + + after_initialize do + self.config = {} if config.nil? rescue nil + end + + before_create do + build_log + self.state = :created if self.state.nil? + self.queue = Queue.for(self).name + end + + after_commit on: :create do + notify(:create) + end + + def propagate(name, *args) + # if we propagate cancel, we can't send it as "cancel", because + # it would trigger cancelling the entire matrix + if name == :cancel + name = :cancel_job + end + Metriks.timer("job.propagate.#{name}").time do + source.send(name, *args) + end + true + end + + def duration + started_at && finished_at ? finished_at - started_at : nil + end + + def ssh_key + config[:source_key] + end + + def config=(config) + super normalize_config(config) + end + + def obfuscated_config + normalize_config(config).deep_dup.tap do |config| + delete_addons(config) + config.delete(:source_key) + if config[:env] + obfuscated_env = process_env(config[:env]) { |env| obfuscate_env(env) } + config[:env] = obfuscated_env ? obfuscated_env.join(' ') : nil + end + if config[:global_env] + obfuscated_env = process_env(config[:global_env]) { |env| obfuscate_env(env) } + config[:global_env] = obfuscated_env ? obfuscated_env.join(' ') : nil + end + end + end + + def decrypted_config + normalize_config(self.config).deep_dup.tap do |config| + config[:env] = process_env(config[:env]) { |env| decrypt_env(env) } if config[:env] + config[:global_env] = process_env(config[:global_env]) { |env| decrypt_env(env) } if config[:global_env] + if config[:addons] + if addons_enabled? + config[:addons] = decrypt_addons(config[:addons]) + else + delete_addons(config) + end + end + end + rescue => e + logger.warn "[job id:#{id}] Config could not be decrypted due to #{e.message}" + {} + end + + def matches_config?(other) + config = self.config.slice(*other.keys) + config = config.merge(branch: commit.branch) if other.key?(:branch) # TODO test this + return false if config.size == 0 + config.all? { |key, value| value == other[key] || commit.branch == other[key] } + end + + def log_content=(content) + create_log! unless log + log.update_attributes!(content: content, aggregated_at: Time.now) + end + + # compatibility, we still use result in webhooks + def result + state.try(:to_sym) == :passed ? 0 : 1 + end + + private + + def delete_addons(config) + if config[:addons].is_a?(Hash) + config[:addons].keep_if { |key, _| WHITELISTED_ADDONS.include? key.to_s } + else + config.delete(:addons) + end + end + + def normalize_config(config) + config = config ? config.deep_symbolize_keys : {} + + if config[:deploy] + if config[:addons].is_a? Hash + config[:addons][:deploy] = config.delete(:deploy) + else + config.delete(:addons) + config[:addons] = { deploy: config.delete(:deploy) } + end + end + + config + end + + def process_env(env) + env = [env] unless env.is_a?(Array) + env = normalize_env(env) + env = if secure_env_enabled? + yield(env) + else + remove_encrypted_env_vars(env) + end + env.compact.presence + end + + def remove_encrypted_env_vars(env) + env.reject do |var| + var.is_a?(Hash) && var.has_key?(:secure) + end + end + + def normalize_env(env) + env.map do |line| + if line.is_a?(Hash) && !line.has_key?(:secure) + line.map { |k, v| "#{k}=#{v}" }.join(' ') + else + line + end + end + end + + def decrypt_addons(addons) + decrypt(addons) + end + + def decrypt_env(env) + env.map do |var| + decrypt(var) do |var| + var.dup.insert(0, 'SECURE ') unless var.include?('SECURE ') + end + end + rescue + {} + end + + def decrypt(v, &block) + repository.key.secure.decrypt(v, &block) + end +end diff --git a/vendor/travis-core/lib/travis/model/job/cleanup.rb b/vendor/travis-core/lib/travis/model/job/cleanup.rb new file mode 100644 index 00000000..609aaa36 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/job/cleanup.rb @@ -0,0 +1,41 @@ +# require 'active_support/concern' +# +# class Job +# +# # Cleanup contains logic that is supposed to re-enqueue stalled jobs +# # and finally finish them forcefully. +# # +# # This stuff is currently not used except when we occasionally +# # re-enqueue jobs manually from the console. +# module Cleanup +# extend ActiveSupport::Concern +# +# FORCE_FINISH_MESSAGE = <<-msg.strip +# This job could not be processed and was forcefully finished. +# msg +# +# included do +# class << self +# def cleanup +# stalled.each do |job| +# job.requeueable? ? job.enqueue : job.force_finish +# end +# end +# +# def stalled +# unfinished.where('created_at < ?', Time.now.utc - Travis.config.jobs.retry.after) +# end +# end +# end +# +# def force_finish +# append_log!("\n#{FORCE_FINISH_MESSAGE}") if respond_to?(:append_log!) +# finish!(state: :errored, finished_at: Time.now.utc) +# end +# +# def requeueable? +# false +# # retries < Travis.config.jobs.retry.max_attempts +# end +# end +# end diff --git a/vendor/travis-core/lib/travis/model/job/queue.rb b/vendor/travis-core/lib/travis/model/job/queue.rb new file mode 100644 index 00000000..ff7b8197 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/job/queue.rb @@ -0,0 +1,121 @@ +class Job + + # Encapsulates logic for figuring out which queue a given job needs to go + # into. + # + # Queue names for Job::Test instances are configured in `Travis.config` and + # are determined based on the repository slug (e.g. 'rails/rails' has its own + # queue) or the language given in the configuration (`.travis.yml`) and + # default to 'builds.linux'. + class Queue + SUDO_REQUIRED_EXECUTABLES = %w( + docker + ping + sudo + ) + + SUDO_DETECTION_REGEXP = /^[^#]*\b(#{SUDO_REQUIRED_EXECUTABLES.join('|')})\b/ + + CUSTOM_STAGES = %w( + before_install + install + before_script + script + before_cache + after_success + after_failure + after_script + before_deploy + ).map(&:to_sym) + + class << self + def for(job) + queues.find(-> { ifnone }) { |queue| queue.matches?(job) } + end + + def queues + @queues ||= Array(Travis.config.queues).compact.map do |queue| + Queue.new(queue[:queue], queue.reject { |key, value| key == :queue }) + end + end + + def default + @default ||= Queue.new(Travis.config.default_queue, {}) + end + + def sudo_detected?(config) + config.values_at(*CUSTOM_STAGES).compact.flatten.any? do |s| + SUDO_DETECTION_REGEXP =~ s.to_s + end + end + + private + + def ifnone + Travis.logger.info("job matches queue #{default.name} via ifnone proc") + default + end + end + + attr_reader :name, :attrs + + def initialize(name, attrs) + @name = name + @attrs = attrs + end + + def matches?(job) + matchers = matchers_for(job) + + unknown_matchers = @attrs.keys - matchers.keys + if unknown_matchers.length > 0 + warn "unknown matchers used for queue #{name}: #{unknown_matchers.join(", ")}" + end + + known_matchers = @attrs.keys & matchers.keys + + all_match = known_matchers.all? do |key| + matchers[key.to_sym] === @attrs[key] + end + + if known_matchers.length > 0 && all_match + logger.info("job matches queue #{name} via matchers #{matchers.inspect}") + return true + end + + false + end + + private + + def matchers_for(job) + { + slug: "#{job.repository.try(:owner_name)}/#{job.repository.try(:name)}", + owner: job.repository.try(:owner_name), + os: job.config[:os], + language: Array(job.config[:language]).flatten.compact.first, + sudo: job.config.fetch(:sudo) { !repo_is_default_docker?(job) }, + dist: job.config[:dist], + group: job.config[:group], + osx_image: job.config[:osx_image], + percentage: lambda { |percentage| rand(100) < percentage }, + services: lambda { |other| !(Array(job.config[:services]) & other).empty? }, + } + end + + def repo_is_default_docker?(job) + return true if Travis::Github::Education.education_queue?(job.repository.try(:owner)) + return false unless Travis::Features.feature_active?(:docker_default_queue) + !self.class.sudo_detected?(job.config) && repo_created_after_docker_cutoff?(job.repository) + end + + def repo_created_after_docker_cutoff?(repository) + return true if repository.created_at.nil? + repository.created_at > Time.parse(Travis.config.docker_default_queue_cutoff) + end + + def logger + Travis.logger + end + end +end diff --git a/vendor/travis-core/lib/travis/model/job/test.rb b/vendor/travis-core/lib/travis/model/job/test.rb new file mode 100644 index 00000000..6977b598 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/job/test.rb @@ -0,0 +1,107 @@ +require 'active_support/core_ext/hash/slice' +require 'simple_states' + +class Job + + # Executes a test job (i.e. runs a test suite) remotely and keeps tabs about + # state changes throughout its lifecycle in the database. + # + # Job::Test belongs to a Build as part of the build matrix and will be + # created with the Build. + class Test < Job + FINISHED_STATES = [:passed, :failed, :errored, :canceled] + FAILED_STATES = [:failed, :errored, :canceled] + + include SimpleStates, Travis::Event + + states :created, :queued, :received, :started, :passed, :failed, :errored, :canceled + + event :receive, to: :received + event :start, to: :started + event :finish, to: :finished + event :reset, to: :created, unless: :created? + event :cancel, to: :canceled, if: :cancelable? + event :all, after: [:propagate, :notify] + + def enqueue # TODO rename to queue and make it an event, simple_states should support that now + update_attributes!(state: :queued, queued_at: Time.now.utc) + notify(:queue) + end + + def receive(data = {}) + log.update_attributes!(content: '', removed_at: nil, removed_by: nil) # TODO this should be in a restart method, right? + data = data.symbolize_keys.slice(:received_at, :worker) + data.each { |key, value| send(:"#{key}=", value) } + end + + def start(data = {}) + data = data.symbolize_keys.slice(:started_at) + data.each { |key, value| send(:"#{key}=", value) } + end + + def finish(data = {}) + data = data.symbolize_keys.slice(:state, :finished_at) + data.each { |key, value| send(:"#{key}=", value) } + end + + def reset(*) + self.state = :created + attrs = %w(started_at queued_at finished_at worker) + attrs.each { |attr| write_attribute(attr, nil) } + if log + log.clear! + else + build_log + end + annotations.destroy_all + end + + def cancel + self.canceled_at = Time.now + self.finished_at = Time.now + + save! + end + + def cancelable? + !finished? + end + + def resetable? + finished? && !invalid_config? + end + + def invalid_config? + config[:".result"] == "parse_error" + end + + def finished? + FINISHED_STATES.include?(state.to_sym) + end + + def finished_unsuccessfully? + FAILED_STATES.include?(state.to_sym) + end + + def passed? + state.to_s == "passed" + end + + def failed? + state.to_s == "failed" + end + + def unknown? + state == nil + end + + def notify(event, *args) + Metriks.timer("job.notify.#{event}").time do + event = :create if event == :reset + super + end + end + + delegate :id, :content, :to => :log, :prefix => true, :allow_nil => true + end +end diff --git a/vendor/travis-core/lib/travis/model/log.rb b/vendor/travis-core/lib/travis/model/log.rb new file mode 100644 index 00000000..66672026 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/log.rb @@ -0,0 +1,54 @@ +require 'metriks' +require 'active_support/core_ext/string/filters' +require 'travis/model' + +class Log < Travis::LogsModel + require 'travis/model/log/part' + + AGGREGATE_PARTS_SELECT_SQL = <<-sql.squish + SELECT array_to_string(array_agg(log_parts.content ORDER BY number, id), '') + FROM log_parts + WHERE log_id = ? + sql + + class << self + def aggregated_content(id) + Metriks.timer('logs.read_aggregated').time do + connection.select_value(sanitize_sql([AGGREGATE_PARTS_SELECT_SQL, id])) || '' + end + end + end + + include Travis::Event + + belongs_to :job + belongs_to :removed_by, class_name: 'User', foreign_key: :removed_by + has_many :parts, class_name: 'Log::Part', foreign_key: :log_id, :dependent => :destroy + + def content + content = read_attribute(:content) || '' + content = [content, self.class.aggregated_content(id)].join unless aggregated? + content + end + + def aggregated? + !!aggregated_at + end + + def clear! + update_column(:content, '') # TODO why in the world does update_attributes not set content to '' + update_column(:aggregated_at, nil) # TODO why in the world does update_attributes not set aggregated_at to nil? + update_column(:archived_at, nil) + update_column(:archive_verified, nil) + Log::Part.where(log_id: id).delete_all + parts.reload + end + + def archived? + archived_at && archive_verified? + end + + def to_json + { 'log' => attributes.slice(*%w(id content created_at job_id updated_at)) }.to_json + end +end diff --git a/vendor/travis-core/lib/travis/model/logs_model.rb b/vendor/travis-core/lib/travis/model/logs_model.rb new file mode 100644 index 00000000..f32209b8 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/logs_model.rb @@ -0,0 +1,3 @@ +class Travis::LogsModel < ActiveRecord::Base + self.abstract_class = true +end diff --git a/vendor/travis-core/lib/travis/model/membership.rb b/vendor/travis-core/lib/travis/model/membership.rb new file mode 100644 index 00000000..5a1900be --- /dev/null +++ b/vendor/travis-core/lib/travis/model/membership.rb @@ -0,0 +1,7 @@ +require 'travis/model' + +class Membership < Travis::Model + belongs_to :user + belongs_to :organization +end + diff --git a/vendor/travis-core/lib/travis/model/organization.rb b/vendor/travis-core/lib/travis/model/organization.rb new file mode 100644 index 00000000..42a4c430 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/organization.rb @@ -0,0 +1,14 @@ +require 'gh' +require 'travis/model' + +class Organization < Travis::Model + has_many :memberships + has_many :users, :through => :memberships + has_many :repositories, :as => :owner + + def education? + Travis::Features.owner_active?(:educational_org, self) + end + alias education education? +end + diff --git a/vendor/travis-core/lib/travis/model/permission.rb b/vendor/travis-core/lib/travis/model/permission.rb new file mode 100644 index 00000000..53ec486e --- /dev/null +++ b/vendor/travis-core/lib/travis/model/permission.rb @@ -0,0 +1,26 @@ +require 'core_ext/active_record/none_scope' +require 'travis/model' + +class Permission < Travis::Model + ROLES = %w(admin push pull) + + class << self + def by_roles(roles) + roles = Array(roles).select { |role| ROLES.include?(role.to_s) } + roles.empty? ? none : where(has_roles(roles)) + end + + def has_roles(roles) + roles.inject(has_role(roles.shift)) do |sql, role| + sql.or(has_role(role)) + end + end + + def has_role(role) + arel_table[role].eq(true) + end + end + + belongs_to :user + belongs_to :repository +end diff --git a/vendor/travis-core/lib/travis/model/repository.rb b/vendor/travis-core/lib/travis/model/repository.rb new file mode 100644 index 00000000..cc70423d --- /dev/null +++ b/vendor/travis-core/lib/travis/model/repository.rb @@ -0,0 +1,206 @@ +require 'uri' +require 'core_ext/hash/compact' +require 'travis/model' + +# Models a repository that has many builds and requests. +# +# A repository has an ssl key pair that is used to encrypt and decrypt +# sensitive data contained in the public `.travis.yml` file, such as Campfire +# authentication data. +# +# A repository also has a ServiceHook that can be used to de/activate service +# hooks on Github. +class Repository < Travis::Model + require 'travis/model/repository/status_image' + require 'travis/model/repository/settings' + + has_many :commits, dependent: :delete_all + has_many :requests, dependent: :delete_all + has_many :builds, dependent: :delete_all + has_many :events + has_many :permissions, dependent: :delete_all + has_many :users, through: :permissions + + has_one :last_build, class_name: 'Build', order: 'id DESC' + has_one :key, class_name: 'SslKey' + belongs_to :owner, polymorphic: true + + validates :name, presence: true + validates :owner_name, presence: true + + before_create do + build_key + end + + delegate :public_key, to: :key + + class << self + def timeline + active.order('last_build_finished_at IS NULL AND last_build_started_at IS NOT NULL DESC, last_build_started_at DESC NULLS LAST, id DESC') + end + + def with_builds + where(arel_table[:last_build_id].not_eq(nil)) + end + + def administratable + includes(:permissions).where('permissions.admin = ?', true) + end + + def recent + limit(25) + end + + def by_owner_name(owner_name) + without_invalidated.where(owner_name: owner_name) + end + + def by_member(login) + without_invalidated.joins(:users).where(users: { login: login }) + end + + def by_slug(slug) + without_invalidated.where(owner_name: slug.split('/').first, name: slug.split('/').last).order('id DESC') + end + + def search(query) + query = query.gsub('\\', '/') + without_invalidated.where("(repositories.owner_name || chr(47) || repositories.name) ILIKE ?", "%#{query}%") + end + + def active + without_invalidated.where(active: true) + end + + def without_invalidated + where(invalidated_at: nil) + end + + def find_by(params) + if id = params[:repository_id] || params[:id] + find_by_id(id) + elsif params[:github_id] + find_by_github_id(params[:github_id]) + elsif params.key?(:slug) + by_slug(params[:slug]).first + elsif params.key?(:name) && params.key?(:owner_name) + by_slug("#{params[:owner_name]}/#{params[:name]}").first + end + end + + def by_name + Hash[*all.map { |repository| [repository.name, repository] }.flatten] + end + + def counts_by_owner_names(owner_names) + query = %(SELECT owner_name, count(*) FROM repositories WHERE owner_name IN (?) AND invalidated_at IS NULL GROUP BY owner_name) + query = sanitize_sql([query, owner_names]) + rows = connection.select_all(query, owner_names) + Hash[*rows.map { |row| [row['owner_name'], row['count'].to_i] }.flatten] + end + end + + delegate :builds_only_with_travis_yml?, to: :settings + + def admin + @admin ||= Travis.run_service(:find_admin, repository: self) # TODO check who's using this + end + + def slug + @slug ||= [owner_name, name].join('/') + end + + def api_url + "#{Travis.config.github.api_url}/repos/#{slug}" + end + + def source_url + (private? || private_mode?) ? "git@#{source_host}:#{slug}.git": "git://#{source_host}/#{slug}.git" + end + + def private_mode? + source_host != 'github.com' + end + + def source_host + Travis.config.github.source_host || 'github.com' + end + + def branches + self.class.connection.select_values %( + SELECT DISTINCT ON (branch) branch + FROM builds + WHERE builds.repository_id = #{id} + ORDER BY branch DESC + LIMIT 25 + ) + end + + def last_completed_build(branch = nil) + builds.api_and_pushes.last_build_on(state: [:passed, :failed, :errored, :canceled], branch: branch) + end + + def last_build_on(branch) + builds.api_and_pushes.last_build_on(branch: branch) + end + + def build_status(branch) + builds.api_and_pushes.last_state_on(state: [:passed, :failed, :errored, :canceled], branch: branch) + end + + def last_finished_builds_by_branches(limit = 50) + Build.joins(%( + inner join ( + select distinct on (branch) builds.id + from builds + where builds.repository_id = #{id} and builds.event_type = 'push' + order by branch, finished_at desc + ) as last_builds on builds.id = last_builds.id + )).limit(limit).order('finished_at DESC') + end + + def regenerate_key! + ActiveRecord::Base.transaction do + key.destroy unless key.nil? + build_key + save! + end + end + + def settings + @settings ||= begin + instance = Repository::Settings.load(super, repository_id: id) + instance.on_save do + self.settings = instance.to_json + self.save! + end + instance + end + end + + def settings=(value) + if value.is_a?(String) || value.nil? + super(value) + else + super(value.to_json) + end + end + + def users_with_permission(permission) + users.includes(:permissions).where(permissions: { permission => true }).limit(10).all + end + + def reload(*) + @settings = nil + super + end + + def multi_os_enabled? + Travis::Features.enabled_for_all?(:multi_os) || Travis::Features.active?(:multi_os, self) + end + + def dist_group_expansion_enabled? + Travis::Features.enabled_for_all?(:dist_group_expansion) || Travis::Features.active?(:dist_group_expansion, self) + end + +end diff --git a/vendor/travis-core/lib/travis/model/repository/settings.rb b/vendor/travis-core/lib/travis/model/repository/settings.rb new file mode 100644 index 00000000..86060fd7 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/repository/settings.rb @@ -0,0 +1,135 @@ +# encoding: utf-8 +require 'coercible' +require 'travis/settings' +require 'travis/overwritable_method_definitions' +require 'travis/settings/encrypted_value' +require 'openssl' + +class Repository::Settings < Travis::Settings + class EnvVar < Travis::Settings::Model + attribute :id, String + attribute :name, String + attribute :value, Travis::Settings::EncryptedValue + attribute :public, Boolean, default: false + attribute :repository_id, Integer + + validates :name, presence: true + end + + class SshKey < Travis::Settings::Model + class NotAPrivateKeyError < StandardError; end + + attribute :description, String + attribute :value, Travis::Settings::EncryptedValue + attribute :repository_id, Integer + + validates :value, presence: true + validate :validate_correctness + + def validate_correctness + return unless value.decrypt + key = OpenSSL::PKey::RSA.new(value.decrypt, '') + raise NotAPrivateKeyError unless key.private? + rescue OpenSSL::PKey::RSAError, NotAPrivateKeyError + # it seems there is no easy way to check if key + # needs a pass phrase with ruby's openssl bindings, + # that's why we need to manually check that + if value.decrypt.to_s =~ /ENCRYPTED/ + errors.add(:value, :key_with_a_passphrase) + else + errors.add(:value, :not_a_private_key) + end + end + end + + class EnvVars < Collection + model EnvVar + + def public + find_all { |var| var.public? } + end + end + + class TimeoutsValidator < ActiveModel::Validator + def validate(settings) + [:hard_limit, :log_silence].each do |type| + next if valid_timeout?(settings, type) + msg = "Invalid #{type} timout value (allowed: 0 - #{max_value(settings, type)})" + settings.errors.add :"timeout_#{type}", msg + end + end + + private + + def valid_timeout?(settings, type) + value = settings.send(:"timeout_#{type}") + value.nil? || value.to_i > 0 && value.to_i <= max_value(settings, type) + end + + def max_value(settings, type) + config = Travis.config.settings.timeouts + values = config.send(custom_timeouts?(settings) ? :maximums : :defaults) || {} + values[type] + end + + def custom_timeouts?(settings) + Travis::Features.repository_active?(:custom_timeouts, settings.repository_id) + end + end + + attribute :env_vars, EnvVars.for_virtus + + attribute :builds_only_with_travis_yml, Boolean, default: false + attribute :build_pushes, Boolean, default: true + attribute :build_pull_requests, Boolean, default: true + attribute :maximum_number_of_builds, Integer + attribute :ssh_key, SshKey + attribute :timeout_hard_limit + attribute :timeout_log_silence + attribute :api_builds_rate_limit, Integer + + validates :maximum_number_of_builds, numericality: true + + validate :api_builds_rate_limit_restriction + + validates_with TimeoutsValidator + + def maximum_number_of_builds + super || 0 + end + + def restricts_number_of_builds? + maximum_number_of_builds > 0 + rescue => e + false + end + + def timeout_hard_limit + value = super + value == 0 ? nil : value + end + + def timeout_log_silence + value = super + value == 0 ? nil : value + end + + def api_builds_rate_limit + super || nil + end + + def api_builds_rate_limit_restriction + if api_builds_rate_limit.to_i > Travis.config.settings.rate_limit.maximums.api_builds + errors.add(:api_builds_rate_limit, "can't be more than 200") + end + end + + + def repository_id + additional_attributes[:repository_id] + end +end + +class Repository::DefaultSettings < Repository::Settings + include Travis::DefaultSettings +end diff --git a/vendor/travis-core/lib/travis/model/repository/status_image.rb b/vendor/travis-core/lib/travis/model/repository/status_image.rb new file mode 100644 index 00000000..94212590 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/repository/status_image.rb @@ -0,0 +1,61 @@ +class Repository + class StatusImage + RESULTS = { + passed: :passing, + failed: :failing, + errored: :error, + canceled: :canceled + } + + attr_reader :repo, :branch + + def initialize(repo, branch = nil) + @repo = repo + @branch = branch + end + + def result + last_state ? RESULTS[last_state] : :unknown + end + + private + + def cache_enabled? + defined?(@cache_enabled) ? @cache_enabled : @cache_enabled = Travis::Features.feature_active?(:states_cache) + end + + def last_state + @last_state ||= (state_from_cache || state_from_database) + end + + def state_from_cache + return unless repo + return unless cache_enabled? + + cache.fetch_state(repo.id, branch).tap do |result| + if result + Metriks.meter('status-image.cache-hit').mark + else + Metriks.meter('status-image.cache-miss').mark + end + end + rescue Travis::StatesCache::CacheError => e + Travis.logger.error(e.message) + return nil + end + + def state_from_database + return unless repo + + build = repo.last_completed_build(branch) + if build + cache.write(repo.id, build.branch, build) if cache_enabled? + build.state.to_sym + end + end + + def cache + Travis.states_cache + end + end +end diff --git a/vendor/travis-core/lib/travis/model/request.rb b/vendor/travis-core/lib/travis/model/request.rb new file mode 100644 index 00000000..25bed946 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/request.rb @@ -0,0 +1,115 @@ +require 'active_record' +require 'simple_states' +require 'travis/model/encrypted_column' + +# Models an incoming request. The only supported source for requests currently is Github. +# +# The Request will be configured by fetching `.travis.yml` from the Github API +# and needs to be approved based on the configuration. Once approved the +# Request creates a Build. +class Request < Travis::Model + require 'travis/model/request/approval' + require 'travis/model/request/branches' + require 'travis/model/request/pull_request' + require 'travis/model/request/states' + + include States, SimpleStates + + serialize :token, Travis::Model::EncryptedColumn.new(disable: true) + + class << self + def last_by_head_commit(head_commit) + where(head_commit: head_commit).order(:id).last + end + + def older_than(id) + recent.where('id < ?', id) + end + + def recent(limit = 25) + order('id DESC').limit(limit) + end + end + + belongs_to :commit + belongs_to :repository + belongs_to :owner, polymorphic: true + has_many :builds + has_many :events, as: :source + + validates :repository_id, presence: true + + serialize :config + serialize :payload + + def event_type + read_attribute(:event_type) || 'push' + end + + def ref + payload['ref'] if payload + end + + def branch_name + ref.scan(%r{refs/heads/(.*?)$}).flatten.first if ref + end + + def tag_name + ref.scan(%r{refs/tags/(.*?)$}).flatten.first if ref + end + + def api_request? + event_type == 'api' + end + + def pull_request? + event_type == 'pull_request' + end + + def pull_request + @pull_request ||= PullRequest.new(payload && payload['pull_request']) + end + + def pull_request_title + pull_request.title if pull_request? + end + + def pull_request_number + pull_request.number if pull_request? + end + + def head_repo + pull_request.head_repo + end + + def base_repo + pull_request.base_repo + end + + def head_branch + pull_request.head_branch + end + + def base_branch + pull_request.base_branch + end + + def config_url + GH.full_url("repos/#{repository.slug}/contents/.travis.yml?ref=#{commit.commit}").to_s + end + + def same_repo_pull_request? + begin + head_repo && base_repo && head_repo == base_repo + rescue => e + Travis.logger.error("[request:#{id}] Couldn't determine whether pull request is from the same repository: #{e.message}") + false + end + end + + def creates_jobs? + Build::Config::Matrix.new( + Build::Config.new(config).normalize, multi_os: repository.multi_os_enabled?, dist_group_expansion: repository.dist_group_expansion_enabled? + ).expand.size > 0 + end +end diff --git a/vendor/travis-core/lib/travis/model/request/approval.rb b/vendor/travis-core/lib/travis/model/request/approval.rb new file mode 100644 index 00000000..cf47b469 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/request/approval.rb @@ -0,0 +1,130 @@ +class Request + class Approval + attr_reader :request, :repository, :commit + + def initialize(request) + @request = request + @repository = request.repository + @commit = request.commit + end + + def settings + repository.settings + end + + delegate :build_pushes?, :build_pull_requests?, to: :settings + + def accepted? + commit.present? && + !repository.private? && + (!excluded_repository? || included_repository?) && + !skipped? && + !compare_url_too_long? && + enabled_in_settings? + end + + def enabled_in_settings? + request.api_request? || (request.pull_request? ? build_pull_requests? : build_pushes?) + end + + def disabled_in_settings? + !enabled_in_settings? + end + + def branch_accepted? + github_pages_explicitly_enabled? || !github_pages? + end + + def config_accepted? + (travis_yml_present? || allow_builds_without_travis_yml?) + end + + def travis_yml_present? + request.config && request.config['.result'] == 'configured' + end + + def allow_builds_without_travis_yml? + !repository.builds_only_with_travis_yml? + end + + def compare_url_too_long? + commit.compare_url.length > 255 + end + + def approved? + accepted? && request.config.present? && branch_approved? && request.creates_jobs? + end + + def result + approved? ? :accepted : :rejected + end + + def message + if !commit.present? + 'missing commit' + elsif excluded_repository? + 'excluded repository' + elsif skipped? + 'skipped through commit message' + elsif disabled_in_settings? + request.pull_request? ? 'pull requests disabled' : 'pushes disabled' + elsif github_pages? + 'github pages branch' + elsif !branch_approved? || !branch_accepted? + 'branch not included or excluded' + elsif !config_accepted? + '.travis.yml is missing and builds without .travis.yml are disabled' + elsif repository.private? + 'private repository' + elsif !request.creates_jobs? + 'matrix created no jobs' + elsif compare_url_too_long? + 'compare URL too long; branch/tag names may be too long' + elsif request.config.blank? + 'config is missing or contains YAML syntax error' + end + end + + private + + def skipped? + Travis::CommitCommand.new(commit.message).skip? + end + + def github_pages_explicitly_enabled? + request.config && + request.config['branches'] && + request.config['branches'].is_a?(Hash) && + request.config['branches']['only'] && + Array(request.config['branches']['only']).grep(/gh[-_]pages/i) + end + + def github_pages? + commit.branch =~ /gh[-_]pages/i + end + + def excluded_repository? + exclude_rules.any? { |rule| repository.slug =~ rule } + end + + def included_repository? + include_rules.any? { |rule| repository.slug =~ rule } + end + + def include_rules + Travis.config.repository_filter.include.map { |rule| rule.is_a?(Regexp) ? rule : Regexp.new(rule) } + end + + def exclude_rules + Travis.config.repository_filter.exclude.map { |rule| rule.is_a?(Regexp) ? rule : Regexp.new(rule) } + end + + def branch_approved? + branches.included?(commit.branch) && !branches.excluded?(commit.branch) + end + + def branches + @branches ||= Branches.new(request) + end + end +end diff --git a/vendor/travis-core/lib/travis/model/request/branches.rb b/vendor/travis-core/lib/travis/model/request/branches.rb new file mode 100644 index 00000000..87e74de2 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/request/branches.rb @@ -0,0 +1,57 @@ +class Request + + # Logic that figures out whether a branch is in- or excluded (white- or + # blacklisted) by the configuration (`.travis.yml`) + class Branches + attr_reader :request, :commit + + def initialize(request) + @request = request + @commit = request.commit + end + + def included?(branch) + !included || includes?(included, branch) + end + + def excluded?(branch) + excluded && includes?(excluded, branch) + end + + private + + def included + config['only'] + end + + def excluded + config['except'] + end + + def includes?(branches, branch) + branches.any? { |pattern| matches?(pattern, branch) } + end + + def matches?(pattern, branch) + pattern = pattern =~ %r{^/(.*)/$} ? Regexp.new($1) : pattern + pattern === branch + end + + def config + @config ||= case branches = request.config.try(:[], 'branches') + when Array + { :only => branches } + when String + { :only => split(branches) } + when Hash + branches.each_with_object({}) { |(k, v), result| result[k] = split(v) } + else + {} + end + end + + def split(branches) + branches.is_a?(String) ? branches.split(',').map(&:strip) : branches + end + end +end diff --git a/vendor/travis-core/lib/travis/model/request/pull_request.rb b/vendor/travis-core/lib/travis/model/request/pull_request.rb new file mode 100644 index 00000000..2e60d600 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/request/pull_request.rb @@ -0,0 +1,39 @@ +class Request + class PullRequest + Try = Struct.new(:value) do + def method_missing(*args, &block) + Try.new(value.nil? ? nil : value.public_send(*args, &block)) + end + end + + attr_reader :payload + + def initialize(payload) + @payload = Try.new(Hashr.new(payload || {})) + end + + def title + payload.title.value + end + + def number + payload.number.value + end + + def head_repo + payload.head.repo.full_name.value + end + + def base_repo + payload.base.repo.full_name.value + end + + def head_branch + payload.head.ref.value + end + + def base_branch + payload.base.ref.value + end + end +end diff --git a/vendor/travis-core/lib/travis/model/request/states.rb b/vendor/travis-core/lib/travis/model/request/states.rb new file mode 100644 index 00000000..344d9ce0 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/request/states.rb @@ -0,0 +1,119 @@ +require 'active_support/concern' +require 'simple_states' + +class Request + module States + extend ActiveSupport::Concern + include Travis::Event + + included do + include SimpleStates + + states :created, :started, :finished + event :start, :to => :started, :after => :configure + event :configure, :to => :configured, :after => :finish + event :finish, :to => :finished + event :all, :after => :notify + end + + def configure + if !accepted? + Travis.logger.warn("[request:configure] Request not accepted: event_type=#{event_type.inspect} commit=#{commit.try(:commit).inspect} message=#{approval.message.inspect}") + else + self.config = fetch_config.merge(config || {}) + + if branch_accepted? && config_accepted? + Travis.logger.info("[request:configure] Request successfully configured commit=#{commit.commit.inspect}") + else + self.config = nil + Travis.logger.warn("[request:configure] Request not accepted: event_type=#{event_type.inspect} commit=#{commit.try(:commit).inspect} message=#{approval.message.inspect}") + end + end + save! + end + + def finish + if config.blank? + Travis.logger.warn("[request:finish] Request not creating a build: config is blank or contains YAML syntax error, config=#{config.inspect} commit=#{commit.try(:commit).inspect}") + elsif !approved? + Travis.logger.warn("[request:finish] Request not creating a build: not approved commit=#{commit.try(:commit).inspect} message=#{approval.message.inspect}") + elsif parse_error? + Travis.logger.info("[request:finish] Request created but Build and Job automatically errored due to a config parsing error. commit=#{commit.try(:commit).inspect}") + add_parse_error_build + elsif server_error? + Travis.logger.info("[request:finish] Request created but Build and Job automatically errored due to a config server error. commit=#{commit.try(:commit).inspect}") + add_server_error_build + else + add_build_and_notify + Travis.logger.info("[request:finish] Request created a build. commit=#{commit.try(:commit).inspect}") + end + self.result = approval.result + self.message = approval.message + Travis.logger.info("[request:finish] Request finished. result=#{result.inspect} message=#{message.inspect} commit=#{commit.try(:commit).inspect}") + end + + def add_build + builds.create!(:repository => repository, :commit => commit, :config => config, :owner => owner) + end + + def add_build_and_notify + add_build.tap do |build| + build.notify(:created) if Travis.config.notify_on_build_created + end + end + + protected + + delegate :accepted?, :approved?, :branch_accepted?, :config_accepted?, :to => :approval + + def approval + @approval ||= Approval.new(self) + end + + def fetch_config + Travis.run_service(:github_fetch_config, request: self) # TODO move to a service, have it pass the config to configure + end + + def add_parse_error_build + Build.transaction do + build = add_build + job = build.matrix.first + job.start!(started_at: Time.now.utc) + job.log_content = < true, :uniqueness => true + validates :public_key, :presence => true + validates :private_key, :presence => true + + before_validation :generate_keys, :on => :create + + serialize :private_key, Travis::Model::EncryptedColumn.new + + def encode(string) + Base64.encode64(encrypt(string)).strip + end + + def encrypt(string) + build_key.public_encrypt(string) + end + + def decrypt(string) + build_key.private_decrypt(string) + end + + def generate_keys! + self.public_key = self.private_key = nil + generate_keys + end + + def generate_keys + unless public_key && private_key + keys = OpenSSL::PKey::RSA.generate(Travis.config.repository.ssl_key.size) + self.public_key = keys.public_key.to_s + self.private_key = keys.to_pem + end + end + + def encoded_public_key + key = build_key.public_key + ['ssh-rsa ', "\0\0\0\assh-rsa#{sized_bytes(key.e)}#{sized_bytes(key.n)}"].pack('a*m').gsub("\n", '') + end + + def encoded_private_key + [private_key].pack('m').strip + end + + def secure + Travis::SecureConfig.new(self) + end + + private + + def build_key + @build_key ||= OpenSSL::PKey::RSA.new(private_key) + end + + def sized_bytes(value) + bytes = to_byte_array(value.to_i) + [bytes.size, *bytes].pack('NC*') + end + + def to_byte_array(num, *significant) + return significant if num.between?(-1, 0) and significant[0][7] == num[7] + to_byte_array(*num.divmod(256)) + significant + end +end diff --git a/vendor/travis-core/lib/travis/model/token.rb b/vendor/travis-core/lib/travis/model/token.rb new file mode 100644 index 00000000..384e1901 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/token.rb @@ -0,0 +1,25 @@ +require 'securerandom' +require 'travis/model' + +# Tokens used for authenticating requests from Github. +# +# Users can have one or many tokens (even though the current UI only allows for +# one) that they need use on their service hooks. This gives us some security +# that people cannot throw random repositories at Travis CI. +class Token < Travis::Model + belongs_to :user + + validate :token, :presence => true + + before_validation :generate_token, on: :create + + attr_accessible # nothing is changable + + serialize :token, Travis::Model::EncryptedColumn.new(disable: true) + + protected + + def generate_token + self.token = SecureRandom.base64(15).tr('+/=lIO0', 'pqrsxyz') + end +end diff --git a/vendor/travis-core/lib/travis/model/url.rb b/vendor/travis-core/lib/travis/model/url.rb new file mode 100644 index 00000000..6d29dfad --- /dev/null +++ b/vendor/travis-core/lib/travis/model/url.rb @@ -0,0 +1,23 @@ +require 'digest/sha1' +require 'travis/model' + +class Url < Travis::Model + validates :url, :presence => true, :uniqueness => true + validates :code, :presence => true, :uniqueness => true + + before_validation :set_code, :on => :create + + def self.shorten(url) + find_or_create_by_url(url) + end + + def short_url + ["http://#{Travis.config.shorten_host}", code].join('/') + end + + private + + def set_code + self.code = Digest::SHA1.hexdigest(url)[0..9] + end +end diff --git a/vendor/travis-core/lib/travis/model/user.rb b/vendor/travis-core/lib/travis/model/user.rb new file mode 100644 index 00000000..f356632b --- /dev/null +++ b/vendor/travis-core/lib/travis/model/user.rb @@ -0,0 +1,150 @@ +require 'gh' +require 'travis/model' + +class User < Travis::Model + require 'travis/model/user/oauth' + + has_many :tokens, dependent: :destroy + has_many :memberships, dependent: :destroy + has_many :organizations, through: :memberships + has_many :permissions, dependent: :destroy + has_many :repositories, through: :permissions + has_many :emails, dependent: :destroy + + attr_accessible :name, :login, :email, :github_id, :github_oauth_token, :gravatar_id, :locale, :education, :first_logged_in_at + + before_create :set_as_recent + after_create :create_a_token + after_commit :sync, on: :create + + serialize :github_scopes + before_save :track_github_scopes + + serialize :github_oauth_token, Travis::Model::EncryptedColumn.new + + class << self + def with_permissions(permissions) + where(:permissions => permissions).includes(:permissions) + end + + def authenticate_by(options) + options = options.symbolize_keys + + if user = User.find_by_login(options[:login]) + user if user.tokens.any? { |t| t.token == options[:token] } + end + end + + def find_or_create_for_oauth(payload) + Oauth.find_or_create_by(payload) + end + + def with_github_token + where("github_oauth_token IS NOT NULL and github_oauth_token != ''") + end + + def with_email(email_address) + Email.where(email: email_address).first.try(:user) + end + end + + def to_json + keys = %w/id login email name locale github_id gravatar_id is_syncing synced_at updated_at created_at/ + { 'user' => attributes.slice(*keys) }.to_json + end + + def permission?(roles, options = {}) + roles, options = nil, roles if roles.is_a?(Hash) + scope = permissions.where(options) + scope = scope.by_roles(roles) if roles + scope.any? + end + + def first_sync? + synced_at.nil? + end + + def sync + Travis.run_service(:sync_user, self) # TODO remove once apps use the service + end + + def syncing? + is_syncing? + end + + def service_hook(options = {}) + service_hooks(options).first + end + + def service_hooks(options = {}) + hooks = repositories + unless options[:all] + hooks = hooks.administratable + end + hooks = hooks.includes(:permissions). + select('repositories.*, permissions.admin as admin, permissions.push as push'). + order('owner_name, name') + # TODO remove owner_name/name once we're on api everywhere + if options.key?(:id) + hooks = hooks.where(options.slice(:id)) + elsif options.key?(:owner_name) || options.key?(:name) + hooks = hooks.where(options.slice(:id, :owner_name, :name)) + end + hooks + end + + def organization_ids + @organization_ids ||= memberships.map(&:organization_id) + end + + def repository_ids + @repository_ids ||= permissions.map(&:repository_id) + end + + def recently_signed_up? + @recently_signed_up || false + end + + def profile_image_hash + # TODO: + # If Github always sends valid gravatar_id in oauth payload (need to check that) + # then these fallbacks (email hash and zeros) are superfluous and can be removed. + gravatar_id.presence || (email? && Digest::MD5.hexdigest(email)) || '0' * 32 + end + + def github_scopes + return [] unless github_oauth_token + read_attribute(:github_scopes) || [] + end + + def correct_scopes? + missing = Oauth.wanted_scopes - github_scopes + missing.empty? + end + + def avatar_url + "https://0.gravatar.com/avatar/#{profile_image_hash}" + end + + def inspect + if github_oauth_token + super.gsub(github_oauth_token, '[REDACTED]') + else + super + end + end + + protected + + def track_github_scopes + self.github_scopes = Travis::Github.scopes_for(self) if github_oauth_token_changed? or github_scopes.blank? + end + + def set_as_recent + @recently_signed_up = true + end + + def create_a_token + self.tokens.create! + end +end diff --git a/vendor/travis-core/lib/travis/model/user/oauth.rb b/vendor/travis-core/lib/travis/model/user/oauth.rb new file mode 100644 index 00000000..17ccf6d9 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/user/oauth.rb @@ -0,0 +1,28 @@ +class User + module Oauth + class << self + def wanted_scopes + return [] unless Travis.config.oauth2.scope + @wanted_scopes ||= Travis.config.oauth2.scope.split(',').sort + end + + def find_or_create_by(payload) + attrs = attributes_from(payload) + user = User.find_by_github_id(attrs['github_id']) + user ? user.update_attributes(attrs) : user = User.create!(attrs) + user + end + + def attributes_from(payload) + { + 'name' => payload['info']['name'], + 'email' => payload['info']['email'], + 'login' => payload['info']['nickname'], + 'github_id' => payload['uid'].to_i, + 'github_oauth_token' => payload['credentials']['token'], + 'gravatar_id' => payload['extra']['raw_info']['gravatar_id'] + } + end + end + end +end diff --git a/vendor/travis-core/lib/travis/model/user/renaming.rb b/vendor/travis-core/lib/travis/model/user/renaming.rb new file mode 100644 index 00000000..c4d8d680 --- /dev/null +++ b/vendor/travis-core/lib/travis/model/user/renaming.rb @@ -0,0 +1,21 @@ +module User::Renaming + def nullify_logins(github_id, login) + users = User.where(["github_id <> ? AND login = ?", github_id, login]) + if users.exists? + Travis.logger.info("About to nullify login (#{login}) for users: #{users.map(&:id).join(', ')}") + users.update_all(login: nil) + end + + organizations = Organization.where(["login = ?", login]) + if organizations.exists? + Travis.logger.info("About to nullify login (#{login}) for organizations: #{organizations.map(&:id).join(', ')}") + organizations.update_all(login: nil) + end + end + + def rename_repos_owner(old_login, new_login) + return if old_login == new_login + Repository.where(owner_name: old_login). + update_all(owner_name: new_login) + end +end diff --git a/vendor/travis-core/lib/travis/notification.rb b/vendor/travis-core/lib/travis/notification.rb new file mode 100644 index 00000000..e5ed94eb --- /dev/null +++ b/vendor/travis-core/lib/travis/notification.rb @@ -0,0 +1,24 @@ +require 'active_support/core_ext/hash/reverse_merge' + +module Travis + module Notification + require 'travis/notification/instrument' + require 'travis/notification/publisher' + + class << self + attr_accessor :publishers + + def setup(options = { instrumentation: true }) + Travis::Instrumentation.setup if options[:instrumentation] && Travis.config.metrics.reporter + publishers << Publisher::Log.new + publishers << Publisher::Redis.new if Travis::Features.feature_active?(:notifications_publisher_redis) + end + + def publish(event) + publishers.each { |publisher| publisher.publish(event) } + end + end + + self.publishers ||= [] + end +end diff --git a/vendor/travis-core/lib/travis/notification/instrument.rb b/vendor/travis-core/lib/travis/notification/instrument.rb new file mode 100644 index 00000000..72160d34 --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/instrument.rb @@ -0,0 +1,58 @@ +require 'active_support/core_ext/object/try' +require 'core_ext/hash/compact' + +module Travis + module Notification + class Instrument + require 'travis/notification/instrument/event_handler' + require 'travis/notification/instrument/task' + + class << self + def attach_to(const) + statuses = %w(received completed failed) + instrumented_methods(const).product(statuses).each do |method, status| + ActiveSupport::Notifications.subscribe(/^#{const.instrumentation_key}(\..+)?.#{method}:#{status}/) do |message, args| + publish(message, method, status, args) + end + end + end + + def instrumented_methods(const) + consts = ancestors.select { |const| (const.name || '')[0..5] == 'Travis' } + methods = consts.map { |const| const.public_instance_methods(false) }.flatten.uniq + methods = methods.map { |method| method.to_s =~ /^(.*)_(received|completed|failed)$/ && $1 } + methods.compact.uniq + end + + def publish(event, method, status, payload) + instrument = new(event, method, status, payload) + callback = :"#{method}_#{status}" + instrument.respond_to?(callback) ? instrument.send(callback) : instrument.publish + end + end + + attr_reader :target, :method, :status, :result, :exception, :meta + + def initialize(event, method, status, payload) + @method, @status = method, status + @target, @result, @exception = payload.values_at(:target, :result, :exception) + started_at, finished_at = payload.values_at(:started_at, :finished_at) + @meta = { + uuid: Travis.uuid, + event: event, + started_at: started_at, + finished_at: finished_at, + duration: finished_at ? finished_at - started_at : nil, + }.compact + end + + def publish(data = {}) + message = "#{target.class.name}##{method}:#{status} #{data.delete(:msg)}".strip + payload = meta.merge(message: message, data: data) + payload[:result] = data.delete(:result) if data.key?(:result) + payload[:exception] = exception if exception + Notification.publish(payload) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/notification/instrument/event_handler.rb b/vendor/travis-core/lib/travis/notification/instrument/event_handler.rb new file mode 100644 index 00000000..512964b0 --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/instrument/event_handler.rb @@ -0,0 +1,35 @@ +require 'travis/notification/instrument' + +module Travis + module Notification + class Instrument + class EventHandler < Instrument + attr_reader :handler, :object, :args, :result + + def initialize(message, method, status, payload) + @handler, @args, @result = payload.values_at(:target, :args, :result) + @object = handler.object + super + end + + def notify_completed + publish + end + + def publish(event = {}) + event = event.reverse_merge( + :msg => "(#{handler.event}) for #<#{object.class.name} id=#{object.id}>", + :object_type => object.class.name, + :object_id => object.id, + :event => handler.event + ) + + event[:payload] = handler.payload + event[:request_id] = object.request_id if object.respond_to?(:request_id) + event[:repository] = object.repository.slug if object.respond_to?(:repository) + super(event) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/notification/instrument/task.rb b/vendor/travis-core/lib/travis/notification/instrument/task.rb new file mode 100644 index 00000000..aa769b09 --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/instrument/task.rb @@ -0,0 +1,34 @@ +module Travis + module Notification + class Instrument + class Task < Instrument + attr_reader :task, :payload + + def initialize(message, method, status, payload) + @task = payload[:target] + @payload = task.payload + super + end + + def run_completed + publish + end + + def publish(event = {}) + event[:msg] = "#{event[:msg]} #{queue_info}" if Travis::Async.enabled? && Travis::Task.run_local? + super(event.merge(:payload => self.payload)) + end + + private + + def queue_info + "(queue size: #{queue.items.size})" if queue + end + + def queue + Travis::Async::Threaded.queues[task.class.name] + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/notification/publisher.rb b/vendor/travis-core/lib/travis/notification/publisher.rb new file mode 100644 index 00000000..4464ca9e --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/publisher.rb @@ -0,0 +1,9 @@ +module Travis + module Notification + module Publisher + require 'travis/notification/publisher/log' + require 'travis/notification/publisher/redis' + require 'travis/notification/publisher/memory' + end + end +end diff --git a/vendor/travis-core/lib/travis/notification/publisher/log.rb b/vendor/travis-core/lib/travis/notification/publisher/log.rb new file mode 100644 index 00000000..a011d344 --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/publisher/log.rb @@ -0,0 +1,42 @@ +module Travis + module Notification + module Publisher + class Log + def publish(event) + return if ignore?(event) + + level = event.key?(:exception) ? :error : :info + message = event[:message] + message = "#{message} (#{'%.5f' % event[:duration]}s)" if event[:duration] + log(level, message) + + if level == :error || Travis.logger.level == ::Logger::DEBUG + event.each do |key, value| + next if key == :message + level = event.key?(:exception) ? :error : :debug + log(level, " #{key}: #{value.inspect}") + end + end + end + + def log(level, msg) + Travis.logger.send(level, msg) + end + + def ignore?(event) + event_received?(event) && !sync_or_request_handler?(event) + end + + def event_received?(event) + event[:event].end_with?("received") + end + + # TODO why do ignore these again? + def sync_or_request_handler?(event) + msg = event[:message] + msg && msg =~ /Travis::Hub::Handler::(Sync|Request)#handle/ + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/notification/publisher/memory.rb b/vendor/travis-core/lib/travis/notification/publisher/memory.rb new file mode 100644 index 00000000..9cb1834c --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/publisher/memory.rb @@ -0,0 +1,15 @@ +module Travis + module Notification + module Publisher + class Memory + def publish(event) + events << event + end + + def events + @events ||= [] + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/notification/publisher/redis.rb b/vendor/travis-core/lib/travis/notification/publisher/redis.rb new file mode 100644 index 00000000..1f53bb2e --- /dev/null +++ b/vendor/travis-core/lib/travis/notification/publisher/redis.rb @@ -0,0 +1,50 @@ +require 'redis' +require 'multi_json' + +module Travis + module Notification + module Publisher + class Redis + extend Exceptions::Handling + attr_accessor :redis, :ttl + + def initialize(options = {}) + @redis = options[:redis] || ::Redis.connect(url: Travis.config.redis.url) + @ttl = options[:ttl] || 10 + end + + def publish(event) + event = filter(event) + payload = MultiJson.encode(event) + # list = 'events:' << event[:uuid] + list = 'events' + + redis.publish list, payload + + # redis.pipelined do + # redis.publish list, payload + # redis.multi do + # redis.persist(list) + # redis.rpush(list, payload) + # redis.expire(list, ttl) + # end + # end + end + rescues :publish, from: Exception + + def filter(value) + case value + when Array + value.map { |value| filter(value) } + when Hash + value.inject({}) { |hash, (key, value)| hash.merge(key => filter(value)) } + when String, Numeric, TrueClass, FalseClass, NilClass + value + else + nil + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/overwritable_method_definitions.rb b/vendor/travis-core/lib/travis/overwritable_method_definitions.rb new file mode 100644 index 00000000..9225e9ed --- /dev/null +++ b/vendor/travis-core/lib/travis/overwritable_method_definitions.rb @@ -0,0 +1,39 @@ +module Travis + # OverwritableMethodDefinitions module allows to easily define methods which will be + # overwritable in the same class. For example, given such a class: + # + # class Foo + # include Travis::OverwritableMethodDefinitions + # + # define_overwritable_method :foo do + # 'foo' + # end + # + # def foo + # super + '!' + # end + # end + # + # Foo.new.foo #=> foo! + module OverwritableMethodDefinitions + def self.included(base) + base.extend(ClassMethods) + base.initialize_overwritable_methods_module + end + + module ClassMethods + def inherited(child) + child.initialize_overwritable_methods_module + end + + def initialize_overwritable_methods_module + @generated_overwritable_methods = Module.new + include @generated_overwritable_methods + end + + def define_overwritable_method(*args, &block) + @generated_overwritable_methods.send :define_method, *args, &block + end + end + end +end diff --git a/vendor/travis-core/lib/travis/redis_pool.rb b/vendor/travis-core/lib/travis/redis_pool.rb new file mode 100644 index 00000000..2db49ff9 --- /dev/null +++ b/vendor/travis-core/lib/travis/redis_pool.rb @@ -0,0 +1,32 @@ +require 'connection_pool' +require 'redis' +require 'metriks' + +module Travis + class RedisPool + attr_reader :pool + + def initialize(options = {}) + pool_options = options.delete(:pool) || {} + pool_options[:size] ||= 10 + pool_options[:timeout] ||= 10 + @pool = ConnectionPool.new(pool_options) do + ::Redis.new(options) + end + end + + def method_missing(name, *args, &block) + timer = Metriks.timer('redis.pool.wait').time + @pool.with do |redis| + timer.stop + if redis.respond_to?(name) + Metriks.timer("redis.operations").time do + redis.send(name, *args, &block) + end + else + super + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/requests.rb b/vendor/travis-core/lib/travis/requests.rb new file mode 100644 index 00000000..2ff1043b --- /dev/null +++ b/vendor/travis-core/lib/travis/requests.rb @@ -0,0 +1,6 @@ +module Travis + module Requests + require 'travis/requests/services' + end +end + diff --git a/vendor/travis-core/lib/travis/requests/services.rb b/vendor/travis-core/lib/travis/requests/services.rb new file mode 100644 index 00000000..8ea5f674 --- /dev/null +++ b/vendor/travis-core/lib/travis/requests/services.rb @@ -0,0 +1,13 @@ +module Travis + module Requests + module Services + require 'travis/requests/services/receive' + + class << self + def register + constants(false).each { |name| const_get(name) } + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/requests/services/receive.rb b/vendor/travis-core/lib/travis/requests/services/receive.rb new file mode 100644 index 00000000..b0f55dab --- /dev/null +++ b/vendor/travis-core/lib/travis/requests/services/receive.rb @@ -0,0 +1,207 @@ +require 'gh' +require 'travis/services/base' +require 'travis/model/request/approval' +require 'travis/notification/instrument' +require 'travis/advisory_locks' +require 'travis/travis_yml_stats' + +module Travis + module Requests + module Services + class Receive < Travis::Services::Base + require 'travis/requests/services/receive/api' + require 'travis/requests/services/receive/cron' + require 'travis/requests/services/receive/pull_request' + require 'travis/requests/services/receive/push' + + extend Travis::Instrumentation + + class PayloadValidationError < StandardError; end + + register :receive_request + + class << self + def payload_for(type, data) + data = GH.load(data) + const_get(type.camelize).new(data) + end + end + + attr_reader :request, :accepted + + def run + with_transactional_advisory_lock do + if accept? + create && start + store_config_info if verify + else + rejected + end + request + end + rescue GH::Error => e + Travis.logger.error "payload for #{slug} could not be received as GitHub returned a #{e.info[:response_status]}: #{e.info}, github-guid=#{github_guid}, event-type=#{event_type}" + end + instrument :run + + def accept? + payload.validate! + validate! + @accepted = payload.accept? + rescue PayloadValidationError => e + Travis.logger.error "#{e.message}, github-guid=#{github_guid}, event-type=#{event_type}" + @accepted = false + end + + private + + def with_transactional_advisory_lock + return yield unless payload.repository + result = nil + Travis::AdvisoryLocks.exclusive("receive-repo:#{payload.repository[:github_id]}", 300) do + ActiveRecord::Base.connection.transaction do + result = yield + end + end + result + rescue => e + ActiveRecord::Base.connection.rollback_db_transaction + raise + end + + def validate! + repo_not_found! unless repo + verify_owner + end + + def verify_owner + owner = owner_by_payload + owner_not_found! unless owner + update_owner(owner) if owner.id != repo.owner_id && !api_request? + end + + def create + @request = repo.requests.create!(payload.request.merge( + :payload => params[:payload], + :event_type => event_type, + :state => :created, + :commit => commit, + :owner => repo.owner, + :token => params[:token] + )) + end + + def start + request.start! + end + + def verify + request.reload + if request.builds.count == 0 + approval = Request::Approval.new(request) + Travis.logger.warn("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} didn't create any builds: #{approval.result}/#{approval.message}") + false + elsif !request.creates_jobs? + approval = Request::Approval.new(request) + Travis.logger.warn("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} didn't create any job: #{approval.result}/#{approval.message}") + false + else + Travis.logger.info("[request:receive] Request #{request.id} commit=#{request.commit.try(:commit).inspect} created #{request.builds.count} builds") + true + end + end + + def update_owner(owner) + repo.update_attributes!(owner: owner, owner_name: owner.login) + owner_updated + end + + def owner_by_payload + if id = payload.repository[:owner_id] + lookup_owner(payload.repository[:owner_type], id: id) + elsif github_id = payload.repository[:owner_github_id] + lookup_owner(payload.repository[:owner_type], github_id: github_id) + elsif login = payload.repository[:owner_name] + lookup_owner(%w(User Organization), login: login) + end + end + + def lookup_owner(types, attrs) + Array(types).map(&:constantize).each do |type| + owner = type.where(attrs).first + return owner if owner + end + nil + end + + def repo_not_found! + Travis::Metrics.meter('request.receive.repository_not_found') + raise PayloadValidationError, "Repository not found: #{payload.repository.slice(:id, :github_id, :owner_name, :name)}" + end + + def owner_not_found! + Travis::Metrics.meter('request.receive.repository_owner_not_found') + raise PayloadValidationError, "The given repository owner could not be found: #{payload.repository.slice(:owner_id, :owner_github_id, :owner_type, :owner_name).inspect}" + end + + def owner_updated + Travis::Metrics.meter('request.receive.update_owner') + Travis.logger.warn("[request:receive] Repository owner updated for #{slug}: #{repo.owner_type}##{repo.owner_id} (#{repo.owner_name})") + end + + def rejected + commit = payload.commit['commit'].inspect if payload.commit rescue nil + Travis.logger.info("[request:receive] Github event rejected: event_type=#{event_type.inspect} repo=\"#{slug}\" commit=#{commit} action=#{payload.action.inspect}") + end + + def payload + @payload ||= self.class.payload_for(event_type, params[:payload]) + end + + def github_guid + params[:github_guid] + end + + def event_type + @event_type ||= (params[:event_type] || 'push').gsub('-', '_') + end + + def api_request? + event_type == 'api' + end + + def repo + @repo ||= run_service(:find_repo, payload.repository) + end + + def slug + payload.repository ? payload.repository.values_at(:owner_name, :name).join('/') : '?' + end + + def commit + @commit ||= repo.commits.create!(payload.commit) if payload.commit + end + + def store_config_info + Travis::TravisYmlStats.store_stats(request) + rescue => e + Travis.logger.warn("[request:receive] Couldn't store .travis.yml stats: #{e.message}") + Travis::Exceptions.handle(e) + end + + class Instrument < Notification::Instrument + def run_completed + params = target.params + publish( + :msg => "type=#{params[:event_type].inspect}", + :type => params[:event_type], + :accept? => target.accepted, + :payload => params[:payload] + ) + end + end + Instrument.attach_to(self) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/api.rb b/vendor/travis-core/lib/travis/requests/services/receive/api.rb new file mode 100644 index 00000000..1711b622 --- /dev/null +++ b/vendor/travis-core/lib/travis/requests/services/receive/api.rb @@ -0,0 +1,106 @@ +module Travis + module Requests + module Services + class Receive < Travis::Services::Base + class Api + VALIDATION_ERRORS = { + repo: 'Repository data is not present in payload', + } + attr_reader :event + + def initialize(event) + @event = event + end + + def accept? + true + end + + def validate! + error(:repo) if repo_data.nil? + end + + def action + nil + end + + def repository + @repository ||= { + owner_id: repo_data['owner_id'], + owner_type: repo_data['owner_type'], + owner_name: repo_data['owner_name'], + name: repo_data['name'] + } + end + + def request + @request ||= { + :config => event['config'] + } + end + + def commit + @commit ||= { + commit: commit_data['sha'], + message: message, + branch: branch, + ref: nil, # TODO verify that we do not need this + committed_at: commit_data['commit']['committer']['date'], # TODO in case of API requests we'd want to display the timestamp of the incoming request + committer_name: commit_data['commit']['committer']['name'], + committer_email: commit_data['commit']['committer']['email'], + author_name: commit_data['commit']['author']['name'], + author_email: commit_data['commit']['author']['email'], + compare_url: commit_data['_links']['self']['href'] + } + end + + private + + def gh + Github.authenticated(user) + end + + def user + @user ||= User.find(event['user']['id']) + end + + def repo_data + event['repository'] || {} + end + + def message + event['message'] || commit_data['commit']['message'] + end + + def slug + repo_data.values_at('owner_name', 'name').join('/') + end + + def branch + event['branch'] || 'master' + end + + def repo_github_id + repo.try(:github_id) || raise(ActiveRecord::RecordNotFound) + end + + def repo + if id = repo_data['id'] + Repository.find(id) + else + Repository.by_slug(slug).first + end + end + + def commit_data + @commit_data ||= gh["repos/#{slug}/commits?sha=#{branch}&per_page=1"].first # TODO I guess Api would protect against GH errors? + end + + def error(type) + raise PayloadValidationError, VALIDATION_ERRORS[type] + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/cron.rb b/vendor/travis-core/lib/travis/requests/services/receive/cron.rb new file mode 100644 index 00000000..dfc3c743 --- /dev/null +++ b/vendor/travis-core/lib/travis/requests/services/receive/cron.rb @@ -0,0 +1,10 @@ +module Travis + module Requests + module Services + class Receive < Travis::Services::Base + class Cron < Api + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb b/vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb new file mode 100644 index 00000000..d09c5c0c --- /dev/null +++ b/vendor/travis-core/lib/travis/requests/services/receive/pull_request.rb @@ -0,0 +1,117 @@ +module Travis + module Requests + module Services + class Receive < Travis::Services::Base + class PullRequest + attr_reader :event + + def initialize(event) + @event = event + end + + def accept? + return false if disabled? || closed? + case action + when :opened, :reopened then !!merge_commit + when :synchronize then head_change? + else false + end + end + + def validate! + if event['repository'].nil? + raise PayloadValidationError, "Repository data is not present in payload" + end + end + + def disabled? + Travis::Features.feature_deactivated?(:pull_requests) + end + + def closed? + pull_request['state'] == 'closed' + end + + def head_change? + head_commit && ::Request.last_by_head_commit(head_commit['sha']).nil? + end + + def repository + @repository ||= repo && { + name: repo['name'], + description: repo['description'], + url: repo['_links']['html']['href'], + owner_github_id: repo['owner']['id'], + owner_type: repo['owner']['type'], + owner_name: repo['owner']['login'], + owner_email: repo['owner']['email'], + private: !!repo['private'], + github_id: repo['id'] + } + end + + def request + @request ||= { + comments_url: comments_url, + base_commit: base_commit['sha'], + head_commit: head_commit['sha'] + } + end + + def commit + @commit ||= if merge_commit + { + commit: merge_commit['sha'], + message: head_commit['message'], + branch: pull_request['base']['ref'], + ref: merge_commit['ref'], + committed_at: committer['date'], + committer_name: committer['name'], + committer_email: committer['email'], + author_name: author['name'], + author_email: author['email'], + compare_url: pull_request['_links']['html']['href'] + } + end + end + + def pull_request + event['pull_request'] || {} + end + + def action + event['action'].try(:to_sym) + end + + def comments_url + pull_request.fetch('_links', {}).fetch('comments', {}).fetch('href', '') + end + + def base_commit + pull_request['base_commit'] || { 'sha' => '' } + end + + def head_commit + pull_request['head_commit'] + end + + def merge_commit + pull_request['merge_commit'] + end + + def repo + event['repository'] + end + + def committer + head_commit.fetch('committer', {}) + end + + def author + head_commit.fetch('author', {}) + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/requests/services/receive/push.rb b/vendor/travis-core/lib/travis/requests/services/receive/push.rb new file mode 100644 index 00000000..bab6b2df --- /dev/null +++ b/vendor/travis-core/lib/travis/requests/services/receive/push.rb @@ -0,0 +1,84 @@ +module Travis + module Requests + module Services + class Receive < Travis::Services::Base + class Push + attr_reader :event + + def initialize(event) + @event = event + end + + def accept? + true + end + + def validate! + if event['repository'].nil? + raise PayloadValidationError, "Repository data is not present in payload" + end + end + + def action + nil + end + + def repository + @repository ||= repo_data && { + name: repo_data['name'], + description: repo_data['description'], + url: repo_data['_links']['html']['href'], + owner_github_id: repo_data['owner']['id'], + owner_type: repo_data['owner']['type'], + owner_name: repo_data['owner']['login'], + owner_email: repo_data['owner']['email'], + private: !!repo_data['private'], + github_id: repo_data['id'] + } + end + + def request + @request ||= {} + end + + def commit + @commit ||= commit_data && { + commit: commit_data['sha'], + message: commit_data['message'], + branch: event['ref'].split('/', 3).last, + ref: event['ref'], + committed_at: commit_data['date'], + committer_name: commit_data['committer']['name'], + committer_email: commit_data['committer']['email'], + author_name: commit_data['author']['name'], + author_email: commit_data['author']['email'], + compare_url: event['compare'] + } + end + + private + + def repo_data + event['repository'] + end + + def commit_data + last_unskipped_commit || commits.last || event['head_commit'] + end + + def last_unskipped_commit + commits.reverse.find { |commit| !skip_commit?(commit) } + end + + def commits + event['commits'] || [] + end + + def skip_commit?(commit) + Travis::CommitCommand.new(commit['message']).skip? + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/secure_config.rb b/vendor/travis-core/lib/travis/secure_config.rb new file mode 100644 index 00000000..232b21ce --- /dev/null +++ b/vendor/travis-core/lib/travis/secure_config.rb @@ -0,0 +1,71 @@ +require 'base64' + +module Travis + # Decrypts a single configuration value from a configuration file using the + # repository's SSL key. + # + # This is used so people can add encrypted sensitive data to their + # `.travis.yml` file. + class SecureConfig < Struct.new(:key) + def self.decrypt(config, key) + self.new(key).decrypt(config) + end + + def self.encrypt(config, key) + self.new(key).encrypt(config) + end + + def decrypt(config) + return config if config.is_a?(String) + + config.inject(config.class.new) do |result, element| + key, element = element if result.is_a?(Hash) + value = process(result, key, decrypt_element(key, element)) + block_given? ? yield(value) : value + end + end + + def encrypt(config) + { 'secure' => key.encode(config) } + end + + private + + def decrypt_element(key, element) + if element.is_a?(Array) || element.is_a?(Hash) + decrypt(element) + elsif secure_key?(key) && element + decrypt_value(element) + else + element + end + end + + def process(result, key, value) + if result.is_a?(Array) + result << value + elsif result.is_a?(Hash) && !secure_key?(key) + result[key] = value + result + else + value + end + end + + def decrypt_value(value) + decoded = Base64.decode64(value) + # TODO should probably be checked earlier + if key.respond_to?(:decrypt) + key.decrypt(decoded) + else + Travis.logger.error("Can not decrypt secure config value: #{value.inspect[0..10]} using key: #{key.inspect[0..10]}") + end + rescue OpenSSL::PKey::RSAError => e + value + end + + def secure_key?(key) + key && (key == :secure || key == 'secure') + end + end +end diff --git a/vendor/travis-core/lib/travis/services.rb b/vendor/travis-core/lib/travis/services.rb new file mode 100644 index 00000000..b555482e --- /dev/null +++ b/vendor/travis-core/lib/travis/services.rb @@ -0,0 +1,74 @@ +module Travis + module Services + module Registry + def add(key, const = nil) + if key.is_a?(Hash) + key.each { |key, const| add(key, const) } + else + services[key.to_sym] = const + end + end + + def [](key) + services[key.to_sym] || raise("can not use unregistered service #{key}. known services are: #{services.keys.inspect}") + end + + private + + def services + @services ||= {} + end + end + + extend Registry + + class << self + def register + constants(false).each { |name| const_get(name) } + end + end + end +end + +require 'travis/services/helpers' + +module Travis + extend Services::Helpers +end + +require 'travis/services/base' +require 'travis/services/cancel_job' +require 'travis/services/cancel_build' +require 'travis/services/delete_caches' +require 'travis/services/find_admin' +require 'travis/services/find_annotations' +require 'travis/services/find_branch' +require 'travis/services/find_branches' +require 'travis/services/find_build' +require 'travis/services/find_builds' +require 'travis/services/find_daily_repos_stats' +require 'travis/services/find_daily_tests_stats' +require 'travis/services/find_caches' +require 'travis/services/find_hooks' +require 'travis/services/find_job' +require 'travis/services/find_jobs' +require 'travis/services/find_log' +require 'travis/services/find_repo' +require 'travis/services/find_repos' +require 'travis/services/find_repo_key' +require 'travis/services/find_requests' +require 'travis/services/find_request' +require 'travis/services/find_repo_settings' +require 'travis/services/find_user_accounts' +require 'travis/services/find_user_broadcasts' +require 'travis/services/find_user_permissions' +require 'travis/services/next_build_number' +require 'travis/services/regenerate_repo_key' +require 'travis/services/remove_log' +require 'travis/services/reset_model' +require 'travis/services/sync_user' +require 'travis/services/update_annotation' +require 'travis/services/update_hook' +require 'travis/services/update_job' +require 'travis/services/update_log' +require 'travis/services/update_user' diff --git a/vendor/travis-core/lib/travis/services/base.rb b/vendor/travis-core/lib/travis/services/base.rb new file mode 100644 index 00000000..0f2c21e1 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/base.rb @@ -0,0 +1,28 @@ +require 'travis/services/helpers' + +module Travis + module Services + class Base + def self.register(key) + Travis.services.add(key, self) + end + + include Helpers + + attr_reader :current_user, :params + + def initialize(*args) + @params = args.last.is_a?(Hash) ? args.pop.symbolize_keys : {} + @current_user = args.last + end + + def scope(key) + key.to_s.camelize.constantize + end + + def logger + Travis.logger + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/cancel_build.rb b/vendor/travis-core/lib/travis/services/cancel_build.rb new file mode 100644 index 00000000..26f2a968 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/cancel_build.rb @@ -0,0 +1,78 @@ +require 'travis/services/base' + +module Travis + module Services + class CancelBuild < Base + extend Travis::Instrumentation + + register :cancel_build + + attr_reader :source + + def initialize(*) + super + + @source = params.delete(:source) || 'unknown' + end + + def run + cancel if can_cancel? + end + instrument :run + + def messages + messages = [] + messages << { :notice => 'The build was successfully cancelled.' } if can_cancel? + messages << { :error => 'You are not authorized to cancel this build.' } unless authorized? + messages << { :error => "The build could not be cancelled." } unless build.cancelable? + messages + end + + def cancel + # build may have been retrieved with a :join query, so we need to reset the readonly status + build.send(:instance_variable_set, :@readonly, false) + build.cancel! + publish! + end + + def publish! + # TODO: I think that instead of keeping publish logic in both cancel build + # and cancel job services, we could call cancel_job service for each job + # in the matrix, which would put build in canceled state, even without calling + # cancel! on build explicitly. This may be a better way to handle cancelling + # build + build.matrix.each do |job| + Travis.logger.info("Publishing cancel_job message to worker.commands queue for Job##{job.id}") + publisher.publish(type: 'cancel_job', job_id: job.id, source: source) + end + + end + + def can_cancel? + authorized? && build.cancelable? + end + + def authorized? + current_user.permission?(:pull, :repository_id => build.repository_id) + end + + def build + @build ||= run_service(:find_build, params) + end + + def publisher + Travis::Amqp::FanoutPublisher.new('worker.commands') + end + + class Instrument < Notification::Instrument + def run_completed + publish( + :msg => "for (#{target.current_user.login})", + :result => result + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/cancel_job.rb b/vendor/travis-core/lib/travis/services/cancel_job.rb new file mode 100644 index 00000000..5be04197 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/cancel_job.rb @@ -0,0 +1,73 @@ +require 'travis/notification' +require 'travis/services/base' + +module Travis + module Services + class CancelJob < Base + extend Travis::Instrumentation + + register :cancel_job + + attr_reader :source + + def initialize(*) + super + + @source = params.delete(:source) || 'unknown' + end + + def run + cancel if can_cancel? + end + instrument :run + + def messages + messages = [] + messages << { :notice => 'The job was successfully cancelled.' } if can_cancel? + messages << { :error => 'You are not authorized to cancel this job.' } unless authorized? + messages << { :error => "The job could not be cancelled because it is currently #{job.state}." } unless job.cancelable? + messages + end + + def cancel + # job may have been retrieved with a :join query, so we need to reset the readonly status + job.send(:instance_variable_set, :@readonly, false) + publish! + job.cancel! + end + + def can_cancel? + authorized? && job.cancelable? + end + + def authorized? + current_user.permission?(:pull, :repository_id => job.repository_id) + end + + def job + @job ||= run_service(:find_job, params) + end + + def publish! + Travis.logger.info("Publishing cancel_job message to worker.commands queue for Job##{job.id}") + publisher.publish(type: 'cancel_job', job_id: job.id, source: source) + end + + private + + def publisher + Travis::Amqp::FanoutPublisher.new('worker.commands') + end + + class Instrument < Notification::Instrument + def run_completed + publish( + :msg => "for (#{target.current_user.login})", + :result => result + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/delete_caches.rb b/vendor/travis-core/lib/travis/services/delete_caches.rb new file mode 100644 index 00000000..fd4526dc --- /dev/null +++ b/vendor/travis-core/lib/travis/services/delete_caches.rb @@ -0,0 +1,15 @@ +require 'travis/services/base' + +module Travis + module Services + class DeleteCaches < Base + register :delete_caches + + def run + caches = run_service(:find_caches, params) + caches.each { |c| c.destroy } + caches + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_admin.rb b/vendor/travis-core/lib/travis/services/find_admin.rb new file mode 100644 index 00000000..ea4eac4f --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_admin.rb @@ -0,0 +1,94 @@ +require 'faraday/error' +require 'travis/services/base' + +# TODO extract github specific stuff to a separate service + +module Travis + module Services + class FindAdmin < Base + extend Travis::Instrumentation + include Travis::Logging + + register :find_admin + + def run + if repository + admin = candidates.first + admin || raise_admin_missing + else + error "[github-admin] repository is nil: #{params.inspect}" + raise Travis::RepositoryMissing, "no repository given" + end + end + instrument :run + + def repository + params[:repository] + end + + private + + def candidates + User.with_github_token.with_permissions(:repository_id => repository.id, :admin => true) + end + + def validate(user) + Timeout.timeout(2) do + data = Github.authenticated(user) { repository_data } + if data['permissions'] && data['permissions']['admin'] + user + else + info "[github-admin] #{user.login} no longer has admin access to #{repository.slug}" + update(user, data['permissions']) + false + end + end + rescue Timeout::Error => error + handle_error(user, error) + false + rescue GH::Error => error + handle_error(user, error) + false + end + + def handle_error(user, error) + status = error.info[:response_status] + case status + when 401 + error "[github-admin] token for #{user.login} no longer valid" + user.update_attributes!(:github_oauth_token => "") + when 404 + info "[github-admin] #{user.login} no longer has any access to #{repository.slug}" + update(user, {}) + else + error "[github-admin] error retrieving repository info for #{repository.slug} for #{user.login}: #{error.message}" + end + end + + # TODO should this not be memoized? + def repository_data + data = GH["repos/#{repository.slug}"] + info "[github-admin] could not retrieve data for #{repository.slug}" unless data + data || { 'permissions' => {} } + end + + def update(user, permissions) + user.update_attributes!(:permissions => permissions) + end + + def raise_admin_missing + raise Travis::AdminMissing.new("no admin available for #{repository.slug}") + end + + class Instrument < Notification::Instrument + def run_completed + publish( + msg: "for #{target.repository.slug}: #{result.login}", + result: result + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_annotations.rb b/vendor/travis-core/lib/travis/services/find_annotations.rb new file mode 100644 index 00000000..bdcd3db1 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_annotations.rb @@ -0,0 +1,17 @@ +module Travis + module Services + class FindAnnotations < Base + register :find_annotations + + def run + if params[:ids] + scope(:annotation).where(id: params[:ids]) + elsif params[:job_id] + scope(:annotation).where(job_id: params[:job_id]) + else + [] + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_branch.rb b/vendor/travis-core/lib/travis/services/find_branch.rb new file mode 100644 index 00000000..ab7fdf4b --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_branch.rb @@ -0,0 +1,36 @@ +require 'core_ext/active_record/none_scope' +require 'travis/services/base' + +module Travis + module Services + class FindBranch < Base + register :find_branch + + def run + result + end + + def updated_at + result.updated_at if result + end + + private + + def result + @result ||= params[:id] ? by_id : by_params + end + + def by_id + scope(:build).find(params[:id]) + end + + def by_params + repo.last_build_on params[:branch] if repo and params[:branch] + end + + def repo + @repo ||= run_service(:find_repo, params) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_branches.rb b/vendor/travis-core/lib/travis/services/find_branches.rb new file mode 100644 index 00000000..95916f33 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_branches.rb @@ -0,0 +1,32 @@ +require 'core_ext/active_record/none_scope' +require 'travis/services/base' + +module Travis + module Services + class FindBranches < Base + register :find_branches + + def run + result + end + + private + + def result + @result ||= params[:ids] ? by_ids : by_params + end + + def by_ids + scope(:build).where(:id => params[:ids]) + end + + def by_params + repo ? repo.last_finished_builds_by_branches : scope(:build).none + end + + def repo + @repo ||= run_service(:find_repo, params) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_build.rb b/vendor/travis-core/lib/travis/services/find_build.rb new file mode 100644 index 00000000..da009e7f --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_build.rb @@ -0,0 +1,55 @@ +require 'travis/services/base' + +module Travis + module Services + class FindBuild < Base + register :find_build + + def run + preload(result) if result + end + + def final? + # TODO builds can be requeued, so finished builds are no more final + # result.try(:finished?) + false + end + + def updated_at + max = all_resources.max_by(&:updated_at) + max.updated_at if max.respond_to?(:updated_at) + end + + private + + def all_resources + if result + all = [result, result.commit, result.request, result.matrix.to_a, result.matrix.map(&:annotations)] + all.flatten.find_all { |r| r.updated_at } + else + [] + end + end + + def result + @result ||= load_result + end + + def load_result + columns = scope(:build).column_names + columns -= %w(config) if params[:exclude_config] + columns = columns.map { |c| %Q{"builds"."#{c}"} } + scope(:build).select(columns).find_by_id(params[:id]).tap do |res| + res.config = {} if params[:exclude_config] + end + end + + def preload(build) + ActiveRecord::Associations::Preloader.new(build, [:matrix, :commit, :request]).run + ActiveRecord::Associations::Preloader.new(build.matrix, :log, select: [:id, :job_id, :updated_at]).run + ActiveRecord::Associations::Preloader.new(build.matrix, :annotations).run + build + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_builds.rb b/vendor/travis-core/lib/travis/services/find_builds.rb new file mode 100644 index 00000000..4cf408b8 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_builds.rb @@ -0,0 +1,54 @@ +require 'core_ext/active_record/none_scope' +require 'travis/services/base' + +# v2 builds.all +# build => commit, request, matrix.id + +module Travis + module Services + class FindBuilds < Base + register :find_builds + + def run + preload(result) + end + + private + + def result + @result ||= params[:ids] ? by_ids : by_params + end + + def by_ids + scope(:build).where(:id => params[:ids]) + end + + def by_params + if repo + # TODO :after_number seems like a bizarre api why not just pass an id? pagination style? + builds = repo.builds + builds = builds.by_event_type(params[:event_type]) if params[:event_type] + if params[:number] + builds.where(:number => params[:number].to_s) + else + builds.older_than(params[:after_number]) + end + elsif params[:running] + scope(:build).running.limit(25) + elsif params.nil? || params == {} + scope(:build).recent + else + scope(:build).none + end + end + + def preload(builds) + builds.includes(:commit) + end + + def repo + @repo ||= run_service(:find_repo, params) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_caches.rb b/vendor/travis-core/lib/travis/services/find_caches.rb new file mode 100644 index 00000000..e218ccb5 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_caches.rb @@ -0,0 +1,191 @@ +require 's3' +require 'travis/services/base' +require 'google/apis/storage_v1' + +module Travis + module Services + class FindCaches < Base + register :find_caches + + class S3Wrapper + attr_reader :repository, :s3_object + + def initialize(repository, s3_object) + @repository = repository + @s3_object = s3_object + end + + def source + 'S3' + end + + def last_modified + s3_object.last_modified + end + + def size + Integer(s3_object.size) + end + + def slug + File.basename(s3_object.key, '.tbz') + end + + def branch + s3_object.key[%r{^\d+/(.*)/[^/]+$}, 1] + end + + def destroy + Travis.logger.info "action=delete backend=s3 s3_object=#{s3_object.key}" + s3_object.destroy + end + + def temporary_url + s3_object.temporary_url + end + + def content + s3_object.content + end + end + + class GcsWrapper + attr_reader :storage, :bucket_name, :repository, :cache_object + + def initialize(storage, bucket_name, repository, cache_object) + @storage = storage + @bucket_name = bucket_name + @repository = repository + @cache_object = cache_object + end + + def source + 'GCS' + end + + def last_modified + cache_object.updated + end + + def size + Integer(cache_object.size) + end + + def slug + File.basename(cache_object.name, '.tbz') + end + + def branch + cache_object.name[%r{^\d+/(.*)/[^/]+$}, 1] + end + + def destroy + Travis.logger.info "action=delete backend=gcs bucket_name=#{bucket_name} cache_name=#{cache_object.name}" + storage.delete_object(bucket_name, cache_object.name) + rescue Google::Apis::ClientError + end + + def content + io = StringIO.new + storage.get_object(bucket_name, cache_object.name, download_dest: io) + io.rewind + io.read + end + end + + def run + return [] unless setup? && permission? + c = caches(prefix: prefix) + c.select! { |o| o.slug.include?(params[:match]) } if params[:match] + c + end + + private + + def setup? + return true if entries.any? { |entry| valid?(entry) } + + logger.warn "[services:find-caches] cache settings incomplete" + false + end + + def permission? + current_user.permission?(required_role, repository_id: repo.id) + end + + def required_role + Travis.config.roles.find_cache || "push" + end + + def repo + @repo ||= run_service(:find_repo, params) + end + + def branch + params[:branch].presence + end + + def prefix + prefix = "#{repo.github_id}/" + prefix << branch << '/' if branch + prefix + end + + def caches(options = {}) + if @caches + return @caches + end + + c = [] + + entries.map do |entry| + if config = entry[:s3] + svc = ::S3::Service.new(config.to_h.slice(:secret_access_key, :access_key_id)) + bucket = svc.buckets.find(config.fetch(:bucket_name)) + + next unless bucket + + c += bucket.objects(options).map { |object| S3Wrapper.new(repo, object) } + elsif config = entry[:gcs] + storage = ::Google::Apis::StorageV1::StorageService.new + json_key_io = StringIO.new(config.to_h[:json_key]) + bucket_name = config[:bucket_name] + + storage.authorization = ::Google::Auth::ServiceAccountCredentials.make_creds( + json_key_io: json_key_io, + scope: [ + 'https://www.googleapis.com/auth/devstorage.read_write' + ] + ) + + next unless items = storage.list_objects(bucket_name, prefix: prefix).items + + items.map do |object| + c << GcsWrapper.new(storage, bucket_name, repo, object) + end + end + end + + @caches = c.compact + end + + def entries + collection = Travis.config.to_h.fetch(:cache_options) { [] } + collection = [collection] unless collection.is_a? Array + collection + end + + def valid?(entry) + valid_s3?(entry) or valid_gcs?(entry) + end + + def valid_s3?(entry) + (s3_config = entry[:s3]) && s3_config[:access_key_id] && s3_config[:secret_access_key] && s3_config[:bucket_name] + end + + def valid_gcs?(entry) + (gcs_config = entry[:gcs]) && gcs_config[:json_key] && gcs_config[:bucket_name] + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_daily_repos_stats.rb b/vendor/travis-core/lib/travis/services/find_daily_repos_stats.rb new file mode 100644 index 00000000..4d9232a8 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_daily_repos_stats.rb @@ -0,0 +1,23 @@ +require 'travis/services/base' + +module Travis + module Services + class FindDailyReposStats < Base + register :find_daily_repos_stats + + def run + select scope(:repository). + select(['date(created_at) AS date', 'count(created_at) AS count']). + where('last_build_id IS NOT NULL'). + group('date'). + order('date').to_sql + end + + private + + def select(sql) + ActiveRecord::Base.connection.select_all(sql) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_daily_tests_stats.rb b/vendor/travis-core/lib/travis/services/find_daily_tests_stats.rb new file mode 100644 index 00000000..d1f1adbf --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_daily_tests_stats.rb @@ -0,0 +1,23 @@ +require 'travis/services/base' + +module Travis + module Services + class FindDailyTestsStats < Base + register :find_daily_tests_stats + + def run + select scope(:job). + select(['date(created_at) AS date', 'count(created_at) AS count']). + group('date'). + order('date'). + where(['created_at > ?', 28.days.ago]).to_sql + end + + private + + def select(sql) + ActiveRecord::Base.connection.select_all(sql) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_hooks.rb b/vendor/travis-core/lib/travis/services/find_hooks.rb new file mode 100644 index 00000000..b4ed9763 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_hooks.rb @@ -0,0 +1,13 @@ +require 'travis/services/base' + +module Travis + module Services + class FindHooks < Base + register :find_hooks + + def run + current_user.service_hooks(params) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_job.rb b/vendor/travis-core/lib/travis/services/find_job.rb new file mode 100644 index 00000000..c7a11a5a --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_job.rb @@ -0,0 +1,48 @@ +require 'travis/services/base' + +module Travis + module Services + class FindJob < Base + register :find_job + + def run + preload(result) if result + end + + def final? + # TODO jobs can be requeued, so finished jobs are no more final + # result.try(:finished?) + false + end + + def updated_at + [result].concat(result.annotations).map(&:updated_at).max if result + end + + private + + def result + @result ||= load_result + rescue ActiveRecord::SubclassNotFound => e + Travis.logger.warn "[services:find-job] #{e.message}" + raise ActiveRecord::RecordNotFound + end + + def load_result + columns = scope(:job).column_names + columns -= %w(config) if params[:exclude_config] + columns = columns.map { |c| %Q{"jobs"."#{c}"} } + scope(:job).select(columns).find_by_id(params[:id]).tap do |res| + res.config = {} if params[:exclude_config] + end + end + + def preload(job) + ActiveRecord::Associations::Preloader.new(job, :log).run + ActiveRecord::Associations::Preloader.new(job, :commit).run + ActiveRecord::Associations::Preloader.new(job, :annotations).run + job + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_jobs.rb b/vendor/travis-core/lib/travis/services/find_jobs.rb new file mode 100644 index 00000000..36eea22c --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_jobs.rb @@ -0,0 +1,43 @@ +require 'travis/services/base' + +module Travis + module Services + class FindJobs < Base + register :find_jobs + + def run + preload(result) + end + + private + + def result + @result ||= params[:ids] ? by_ids : by_params + end + + def by_ids + scope(:job).where(:id => params[:ids]) + end + + def by_params + jobs = scope(:job) + if params[:state] + jobs = jobs.where(state: params[:state]) + else + jobs = jobs.where(state: [:created, :queued, :received, :started]) + # we don't use it anymore, but just for backwards compat + jobs = jobs.where(queue: params[:queue]) if params[:queue] + jobs + end + jobs.limit(250) + end + + def preload(jobs) + jobs = jobs.includes(:commit) + ActiveRecord::Associations::Preloader.new(jobs, :log, :select => [:id, :job_id]).run + ActiveRecord::Associations::Preloader.new(jobs, :repository, :select => [:id, :owner_name, :name]).run + jobs + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_log.rb b/vendor/travis-core/lib/travis/services/find_log.rb new file mode 100644 index 00000000..6264e287 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_log.rb @@ -0,0 +1,33 @@ +require 'travis/services/base' + +module Travis + module Services + class FindLog < Base + register :find_log + + def run(options = {}) + result if result + end + + def final? + # TODO jobs can be requeued, so finished jobs are no more final + # result && result.job && result.job.finished? + false + end + + # def updated_at + # result.updated_at + # end + + private + + def result + @result ||= if params[:id] + scope(:log).find_by_id(params[:id]) + elsif params[:job_id] + scope(:log).where(job_id: params[:job_id]).first + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_repo.rb b/vendor/travis-core/lib/travis/services/find_repo.rb new file mode 100644 index 00000000..26cb0fe7 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_repo.rb @@ -0,0 +1,23 @@ +require 'travis/services/base' + +module Travis + module Services + class FindRepo < Base + register :find_repo + + def run(options = {}) + result + end + + def updated_at + result.try(:updated_at) + end + + private + + def result + @result ||= scope(:repository).find_by(params) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_repo_key.rb b/vendor/travis-core/lib/travis/services/find_repo_key.rb new file mode 100644 index 00000000..1c60e555 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_repo_key.rb @@ -0,0 +1,23 @@ +require 'travis/services/base' + +module Travis + module Services + class FindRepoKey < FindRepo + register :find_repo_key + + def run(options = {}) + result + end + + def updated_at + result.try(:updated_at) + end + + private + + def result + @result ||= (repo = super) ? repo.key : nil + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_repo_settings.rb b/vendor/travis-core/lib/travis/services/find_repo_settings.rb new file mode 100644 index 00000000..a7ec4ce6 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_repo_settings.rb @@ -0,0 +1,31 @@ +require 'travis/services/base' + +module Travis + module Services + class FindRepoSettings < Base + register :find_repo_settings + + def run(options = {}) + result if repo && authorized? + end + + def updated_at + result.try(:updated_at) + end + + def authorized? + current_user && current_user.permission?(:push, :repository_id => repo.id) + end + + private + + def repo + @repo ||= run_service(:find_repo, params) + end + + def result + repo.settings if repo + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_repos.rb b/vendor/travis-core/lib/travis/services/find_repos.rb new file mode 100644 index 00000000..3a6bfa4f --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_repos.rb @@ -0,0 +1,53 @@ +require 'travis/services/base' + +module Travis + module Services + class FindRepos < Base + register :find_repos + + def run + result + end + + private + + def result + @result ||= params[:ids] ? by_ids : by_params + end + + def by_ids + scope(:repository).where(:id => params[:ids]) + end + + def by_params + scope = self.scope(:repository).without_invalidated + scope = scope.timeline.recent if timeline? + scope = scope.by_member(params[:member]) if params[:member] + scope = scope.by_owner_name(params[:owner_name]) if params[:owner_name] + scope = scope.by_slug(params[:slug]) if params[:slug] + if params[:search].present? + scope = scope.search(params[:search]).order('last_build_started_at DESC NULLS LAST') + end + scope = scope.limit(limit) if limit + scope + end + + def limit + limit = params[:limit].to_i + + return 25 if limit == 0 + + if limit > 50 + 50 + else + limit + end + end + + def timeline? + # :member is passed for the left sidebar on pro/enterprise + not [:owner_name, :slug, :search].any? { |key| params[key] } + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_request.rb b/vendor/travis-core/lib/travis/services/find_request.rb new file mode 100644 index 00000000..2a9d4cb0 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_request.rb @@ -0,0 +1,25 @@ +module Travis + module Services + class FindRequest < Base + register :find_request + + def run(options = {}) + result + end + + def final? + true + end + + def updated_at + result.updated_at if result.respond_to?(:updated_at) + end + + private + + def result + @result ||= scope(:request).find_by_id(params[:id]) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_requests.rb b/vendor/travis-core/lib/travis/services/find_requests.rb new file mode 100644 index 00000000..eb575767 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_requests.rb @@ -0,0 +1,50 @@ +require 'core_ext/active_record/none_scope' + +module Travis + module Services + class FindRequests < Base + register :find_requests + + def run + preload(result) + end + + private + + def preload(requests) + requests.includes(:commit, :builds) + end + + def result + if repo + columns = %w/id repository_id commit_id created_at owner_id owner_type + event_type base_commit head_commit result message payload state/ + requests = repo.requests.select(columns.map { |c| %Q["requests"."#{c}"] }) + if params[:older_than] + requests.older_than(params[:older_than]) + else + requests.recent(requests_limit) + end + else + raise Travis::RepositoryNotFoundError.new(params) + end + end + + def repo + @repo ||= run_service(:find_repo, params) + end + + def requests_limit + max_limit = Travis.config.services.find_requests.max_limit + default_limit = Travis.config.services.find_requests.default_limit + if !params[:limit] + default_limit + elsif params[:limit] > max_limit + max_limit + else + params[:limit] + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_user_accounts.rb b/vendor/travis-core/lib/travis/services/find_user_accounts.rb new file mode 100644 index 00000000..3f75cd33 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_user_accounts.rb @@ -0,0 +1,33 @@ +require 'travis/services/base' + +module Travis + module Services + class FindUserAccounts < Base + register :find_user_accounts + + def run + ([current_user] + orgs).map do |record| + ::Account.from(record, :repos_count => repos_counts[record.login]) + end + end + + private + + def orgs + Organization.where(:login => account_names) + end + + def repos_counts + @repos_counts ||= Repository.counts_by_owner_names(account_names) + end + + def account_names + repos = current_user.repositories + unless params[:all] + repos = repos.administratable + end + repos.select(:owner_name).map(&:owner_name).uniq + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_user_broadcasts.rb b/vendor/travis-core/lib/travis/services/find_user_broadcasts.rb new file mode 100644 index 00000000..df9c43e1 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_user_broadcasts.rb @@ -0,0 +1,13 @@ +require 'travis/services/base' + +module Travis + module Services + class FindUserBroadcasts < Base + register :find_user_broadcasts + + def run + Broadcast.by_user(current_user) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/find_user_permissions.rb b/vendor/travis-core/lib/travis/services/find_user_permissions.rb new file mode 100644 index 00000000..0e1cf6a0 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/find_user_permissions.rb @@ -0,0 +1,15 @@ +require 'travis/services/base' + +module Travis + module Services + class FindUserPermissions < Base + register :find_user_permissions + + def run + scope = current_user.permissions + scope = scope.by_roles(params[:roles].to_s.split(',')) if params[:roles] + scope + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/helpers.rb b/vendor/travis-core/lib/travis/services/helpers.rb new file mode 100644 index 00000000..65f41a4f --- /dev/null +++ b/vendor/travis-core/lib/travis/services/helpers.rb @@ -0,0 +1,20 @@ +require 'travis/services/registry' + +module Travis + module Services + module Helpers + def run_service(key, *args) + service(key, *args).run + end + + def service(key, *args) + params = args.last.is_a?(Hash) ? args.pop : {} + user = args.last + user ||= current_user if respond_to?(:current_user) + Travis.services[key].new(user, params) + end + end + end + + extend Services::Helpers +end diff --git a/vendor/travis-core/lib/travis/services/next_build_number.rb b/vendor/travis-core/lib/travis/services/next_build_number.rb new file mode 100644 index 00000000..480c984a --- /dev/null +++ b/vendor/travis-core/lib/travis/services/next_build_number.rb @@ -0,0 +1,40 @@ +require 'travis/services/base' +require 'travis/notification' + +module Travis + module Services + class NextBuildNumber < Base + extend Travis::Instrumentation + + register :next_build_number + + def run + number = repository.next_build_number + if number.nil? + number = repository.builds.maximum('number::int4').to_i + 1 + repository.next_build_number = number + 1 + else + repository.next_build_number += 1 + end + repository.save!(validate: false) + number + end + instrument :run + + def repository + @repository ||= Repository.find(params[:repository_id]) + end + + class Instrument < Notification::Instrument + def run_completed + params = target.params + publish( + msg: "for repository_id=#{params[:repository_id]}", + repository_id: params[:repository_id] + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/regenerate_repo_key.rb b/vendor/travis-core/lib/travis/services/regenerate_repo_key.rb new file mode 100644 index 00000000..f58e119a --- /dev/null +++ b/vendor/travis-core/lib/travis/services/regenerate_repo_key.rb @@ -0,0 +1,35 @@ +require 'travis/services/base' + +module Travis + module Services + class RegenerateRepoKey < Base + register :regenerate_repo_key + + def run(options = {}) + if repo && accept? + regenerate + repo.key + end + end + + def accept? + has_permission? + end + + private + + def regenerate + repo.regenerate_key! + end + + def repo + @repo ||= service(:find_repo, params).run + end + + def has_permission? + current_user && current_user.permission?(:admin, :repository_id => repo.id) + end + + end + end +end diff --git a/vendor/travis-core/lib/travis/services/registry.rb b/vendor/travis-core/lib/travis/services/registry.rb new file mode 100644 index 00000000..1632b061 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/registry.rb @@ -0,0 +1,31 @@ +module Travis + module Services + module Registry + def add(key, const = nil) + if key.is_a?(Hash) + key.each { |key, const| add(key, const) } + else + services[key.to_sym] = const + end + end + + def [](key) + services[key.to_sym] || raise("can not use unregistered service #{key}. known services are: #{services.keys.inspect}") + end + + private + + def services + @services ||= {} + end + end + + extend Registry + + class << self + def register + constants(false).each { |name| const_get(name) } + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/remove_log.rb b/vendor/travis-core/lib/travis/services/remove_log.rb new file mode 100644 index 00000000..67aaf426 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/remove_log.rb @@ -0,0 +1,79 @@ +module Travis + module Services + class RemoveLog < Base + extend Travis::Instrumentation + include Travis::Logging + + register :remove_log + + FORMAT = "Log removed by %s at %s" + + def run + return nil unless job + + if log.removed_at || log.removed_by + error "Log for job #{job.id} has already been removed by #{log.removed_by} at #{log.removed_at}" + raise LogAlreadyRemoved, "Log for job #{job.id} has already been removed" + end + + unless authorized? + error "Current user #{current_user} is unauthorized to remove log for job #{job.id}" + raise AuthorizationDenied, "insufficient permission to remove logs" + end + + unless job.finished? + error " is not finished" + raise JobUnfinished, "Job #{job.id} is not finished" + end + + removed_at = Time.now + + message = FORMAT % [current_user.name, removed_at.utc] + if params[:reason] + message << "\n\n#{params[:reason]}" + end + + log.clear! + log.update_attributes!( + :content => nil, + :aggregated_at => nil, + :removed_at => removed_at, + :removed_by => current_user + ) + log.parts.create(content: message, number: 1, final: true) + log + end + + instrument :run + + def log + @log ||= job.log + end + + def can_remove? + authorized? && job.finished? + end + + def authorized? + current_user && current_user.permission?(:push, repository_id: job.repository.id) + end + + def job + @job ||= scope(:job).find_by_id(params[:id]) + rescue ActiveRecord::SubclassNotFound => e + Travis.logger.warn "[services:remove-log] #{e.message}" + raise ActiveRecord::RecordNotFound + end + + class Instrument < Notification::Instrument + def run_completed + publish( + :msg => "for (#{target.current_user.login})", + :result => result + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/reset_model.rb b/vendor/travis-core/lib/travis/services/reset_model.rb new file mode 100644 index 00000000..f8a2a0db --- /dev/null +++ b/vendor/travis-core/lib/travis/services/reset_model.rb @@ -0,0 +1,74 @@ +require 'travis/support/instrumentation' +require 'travis/services/base' + +module Travis + module Services + class ResetModel < Base + extend Travis::Instrumentation + + register :reset_model + + def run + reset if current_user && target && accept? + true + end + instrument :run + + def accept? + current_user && permission? && resetable? + end + + def messages + messages = [] + messages << { notice: "The #{type} was successfully restarted." } if accept? + messages << { error: 'You do not seem to have sufficient permissions.' } unless permission? + messages << { error: "This #{type} currently can not be restarted." } unless resetable? + messages + end + + def type + @type ||= params[:build_id] ? :build : :job + end + + def id + @id ||= params[:"#{type}_id"] + end + + private + + def reset + # target may have been retrieved with a :join query, so we need to reset the readonly status + target.send(:instance_variable_set, :@readonly, false) + target.reset!(reset_matrix: type == :build) + end + + def permission? + current_user.permission?(required_role, repository_id: target.repository_id) + end + + def resetable? + defined?(@resetable) ? @resetable : @resetable = target.resetable? + end + + def required_role + Travis.config.roles.reset_model + end + + def target + @target ||= service(:"find_#{type}", id: id).run + end + + class Instrument < Notification::Instrument + def run_completed + publish( + msg: "build_id=#{target.id} #{target.accept? ? 'accepted' : 'not accepted'}", + type: target.type, + id: target.id, + accept?: target.accept? + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/sync_user.rb b/vendor/travis-core/lib/travis/services/sync_user.rb new file mode 100644 index 00000000..98849f2e --- /dev/null +++ b/vendor/travis-core/lib/travis/services/sync_user.rb @@ -0,0 +1,26 @@ +require 'travis/sidekiq/synchronize_user' +require 'travis/services/base' + +module Travis + module Services + class SyncUser < Base + register :sync_user + + def run + trigger_sync unless user.syncing? + end + + def trigger_sync + logger.info("Synchronizing via Sidekiq for user: #{user.login}") + Travis::Sidekiq::SynchronizeUser.perform_async(user.id) + user.update_column(:is_syncing, true) + true + end + + def user + # TODO check that clients are only passing the id + @user ||= current_user || User.find(params[:id]) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/update_annotation.rb b/vendor/travis-core/lib/travis/services/update_annotation.rb new file mode 100644 index 00000000..94a9c03c --- /dev/null +++ b/vendor/travis-core/lib/travis/services/update_annotation.rb @@ -0,0 +1,36 @@ +module Travis + module Services + class UpdateAnnotation < Base + register :update_annotation + + def run + if annotations_enabled? && annotation_provider + cached_annotation = annotation + cached_annotation.update_attributes!(attributes) + + cached_annotation + end + end + + private + + def annotations_enabled? + job = Job.find(params[:job_id]) + repo = job.repository + Travis::Features.enabled_for_all?(:annotations) || Travis::Features.active?(:annotations, repo) + end + + def annotation + annotation_provider.annotation_for_job(params[:job_id]) + end + + def annotation_provider + @annotation_provider ||= AnnotationProvider.authenticate_provider(params[:username], params[:key]) + end + + def attributes + params.slice(:description, :status, :url) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/services/update_hook.rb b/vendor/travis-core/lib/travis/services/update_hook.rb new file mode 100644 index 00000000..bae79049 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/update_hook.rb @@ -0,0 +1,47 @@ +require 'travis/support/instrumentation' +require 'travis/services/base' + +module Travis + module Services + class UpdateHook < Base + extend Travis::Instrumentation + + register :update_hook + + def run + run_service(:github_set_hook, id: repo.id, active: active?) + repo.update_column(:active, active?) + true + end + instrument :run + + # TODO + # def messages + # messages = [] + # messages << { :notice => "The service hook was successfully #{active? ? 'enabled' : 'disabled'}." } if what? + # messages << { :error => 'The service hook could not be set.' } unless what? + # messages + # end + + def repo + @repo ||= current_user.service_hook(params.slice(:id, :owner_name, :name)) + end + + def active? + active = params[:active] + active = { 'true' => true, 'false' => false }[active] if active.is_a?(String) + !!active + end + + class Instrument < Notification::Instrument + def run_completed + publish( + :msg => "for #{target.repo.slug} active=#{target.active?.inspect} (#{target.current_user.login})", + :result => result + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/update_job.rb b/vendor/travis-core/lib/travis/services/update_job.rb new file mode 100644 index 00000000..30be4ca1 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/update_job.rb @@ -0,0 +1,71 @@ +require 'active_support/core_ext/hash/except' +require 'travis/support/instrumentation' +require 'travis/services/base' + +module Travis + module Services + class UpdateJob < Base + extend Travis::Instrumentation + + register :update_job + + EVENT = [:receive, :start, :finish, :reset] + + def run + if job.canceled? && event != :reset + # job is canceled, so we ignore events other than reset + # and we send cancel event to the worker, it may not get + # the first one + cancel_job_in_worker + else + Metriks.timer("update_job.#{event}").time do + job.send(:"#{event}!", data.except(:id)) + end + end + end + instrument :run + + def job + @job ||= Job::Test.find(data[:id]) + end + + def data + @data ||= begin + data = params[:data].symbolize_keys + # TODO remove once workers send the state + data[:state] = { 0 => :passed, 1 => :failed }[data.delete(:result)] if data.key?(:result) + data + end + end + + def event + @event ||= EVENT.detect { |event| event == params[:event].try(:to_sym) } || raise_unknown_event + end + + def raise_unknown_event + raise ArgumentError, "Unknown event: #{params[:event]}, data: #{data}" + end + + def cancel_job_in_worker + publisher.publish(type: 'cancel_job', job_id: job.id, source: 'update_job_service') + end + + def publisher + Travis::Amqp::FanoutPublisher.new('worker.commands') + end + + class Instrument < Notification::Instrument + def run_completed + publish( + msg: "event: #{target.event} for data=#{target.data.inspect}", + job_id: target.data[:id], + event: target.event, + data: target.data + ) + end + end + Instrument.attach_to(self) + end + end +end + diff --git a/vendor/travis-core/lib/travis/services/update_log.rb b/vendor/travis-core/lib/travis/services/update_log.rb new file mode 100644 index 00000000..530a5253 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/update_log.rb @@ -0,0 +1,31 @@ +require 'travis/support/instrumentation' +require 'travis/services/base' + +module Travis + module Services + class UpdateLog < Base + extend Travis::Instrumentation + + register :update_log + + def run + log = run_service(:find_log, id: params[:id]) + log.update_attributes(archived_at: params[:archived_at], archive_verified: params[:archive_verified]) if log + end + instrument :run + + class Instrument < Notification::Instrument + def run_completed + publish( + msg: "for # params=#{target.params.inspect}", + object_type: 'Log', + object_id: target.params[:id], + params: target.params, + result: result + ) + end + end + Instrument.attach_to(self) + end + end +end diff --git a/vendor/travis-core/lib/travis/services/update_user.rb b/vendor/travis-core/lib/travis/services/update_user.rb new file mode 100644 index 00000000..c0a12ac5 --- /dev/null +++ b/vendor/travis-core/lib/travis/services/update_user.rb @@ -0,0 +1,38 @@ +require 'travis/services/base' + +module Travis + module Services + class UpdateUser < Base + register :update_user + + LOCALES = %w(en es fr ja nb nl pl pt-BR ru de) # TODO how to figure these out + + attr_reader :result + + def run + @result = current_user.update_attributes!(attributes) if valid_locale? + true + end + + def messages + messages = [] + if result + messages << { :notice => "Your profile was successfully updated." } + else + messages << { :error => 'Your profile could not be updated.' } + end + messages + end + + private + + def attributes + params.slice(:locale) + end + + def valid_locale? + LOCALES.include?(params[:locale]) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/settings.rb b/vendor/travis-core/lib/travis/settings.rb new file mode 100644 index 00000000..dcaf9eb9 --- /dev/null +++ b/vendor/travis-core/lib/travis/settings.rb @@ -0,0 +1,67 @@ +require 'coercible' +require 'travis/overwritable_method_definitions' +require 'travis/settings/collection' +require 'travis/settings/model' +require 'travis/settings/model_extensions' + +module Travis + class Settings + include Travis::OverwritableMethodDefinitions + include Virtus.model + include ActiveModel::Validations + include Travis::Settings::ModelExtensions + + def on_save(&block) + @on_save = block + self + end + + def merge(hash) + hash.each { |k, v| + set(k, v) unless collection?(k) || model?(k) + } + end + + def obfuscated + to_hash + end + + def save + if valid? + @on_save.call + end + end + + def to_json + to_hash.to_json + end + + def to_hash + result = super + + result.each do |key, value| + if value.respond_to?(:to_hash) + result[key] = value.to_hash + end + end + + result + end + end + + module DefaultSettings + def initialize(*) + super + + freeze + end + + def merge(*) + raise "merge is not supported on default settings" + end + + def set(key, value) + raise "setting values is not supported on default settings" + end + end +end diff --git a/vendor/travis-core/lib/travis/settings/collection.rb b/vendor/travis-core/lib/travis/settings/collection.rb new file mode 100644 index 00000000..8d378756 --- /dev/null +++ b/vendor/travis-core/lib/travis/settings/collection.rb @@ -0,0 +1,79 @@ +class Travis::Settings + class Collection + include Enumerable + + delegate :each, :<<, :push, :delete, :length, :first, :last, to: '@collection' + attr_accessor :additional_attributes + + class << self + # This feels a bit weird, but I don't know how to do it better. + # Virtus checks for collection type by checking an array member, + # so if you pass Array[String], a collection type will be set to String. + # Here, we already specify what is a model class for a collection. + # In order to not have to specify class twice, I created this method + # which creates just what Virtus needs. + def for_virtus + self[model_class] + end + + def [](*args) + new(*args) + end + + def model(model_name_or_class = nil) + if model_name_or_class + klass = if model_name_or_class.is_a?(String) || model_name_or_class.is_a?(Symbol) + name = model_name_or_class.to_s.classify + self.const_defined?(name, false) ? self.const_get(name, false) : Travis::Settings.const_get(name, false) + else + model_name_or_class + end + + @model_class = klass + else + @model_class + end + end + attr_reader :model_class + end + + delegate :model_class, to: 'self.class' + + def initialize(*args) + @collection = Array[*args] + end + + def create(attributes) + model = model_class.new(attributes) + model.load({}, additional_attributes) + model.id = SecureRandom.uuid unless model.id + push model + model + end + + def find(id) + detect { |model| model.id == id.to_s } + end + + def destroy(id) + record = find(id) + if record + delete record + record + end + end + + def to_hash + @collection.map(&:to_hash) + end + + def load(collection, additional_attributes = {}) + self.additional_attributes = additional_attributes + return unless collection.respond_to?(:each) + + collection.each do |element| + self.push model_class.load(element, additional_attributes) + end + end + end +end diff --git a/vendor/travis-core/lib/travis/settings/encrypted_value.rb b/vendor/travis-core/lib/travis/settings/encrypted_value.rb new file mode 100644 index 00000000..a9b852e6 --- /dev/null +++ b/vendor/travis-core/lib/travis/settings/encrypted_value.rb @@ -0,0 +1,62 @@ +require 'virtus' + +class Travis::Settings + class EncryptedValue + include Virtus.value_object + attr_reader :value, :key + + values do + attribute :value, String + end + + def initialize(value) + if value.is_a? String + # a value is set through the accessor, not loaded in jason, we + # need to encrypt it and put into hash form + value = { value: encrypt(value) } + end + + super value + end + + def to_s + value + end + + def to_str + value + end + + def to_json(*) + as_json.to_json + end + + def as_json(*) + value + end + + def to_hash + value + end + + def inspect + "" + end + + def key + Travis.config.encryption.key + end + + def encrypt(value) + Travis::Model::EncryptedColumn.new(key: key, use_prefix: false).dump(value) + end + + def decrypt + Travis::Model::EncryptedColumn.new(key: key, use_prefix: false).load(value) + end + + def load(value, additional_attributes = nil) + self.instance_variable_set('@value', value) + end + end +end diff --git a/vendor/travis-core/lib/travis/settings/model.rb b/vendor/travis-core/lib/travis/settings/model.rb new file mode 100644 index 00000000..37b97f7c --- /dev/null +++ b/vendor/travis-core/lib/travis/settings/model.rb @@ -0,0 +1,39 @@ +require 'virtus' +require 'travis/settings/encrypted_value' +require 'travis/settings/model_extensions' + +class Travis::Settings + class Model + include Virtus.model + include ActiveModel::Validations + include ModelExtensions + include ActiveModel::Serialization + + def attribute?(name) + attributes.keys.include?(name.to_sym) + end + + def read_attribute_for_serialization(name) + self.send(name) if attribute?(name) + end + + def read_attribute_for_validation(name) + return unless attribute?(name) + + value = self.send(name) + value.is_a?(EncryptedValue) ? value.to_s : value + end + + def update(attributes) + self.attributes = attributes + end + + def key + Travis.config.encryption.key + end + + def to_json + to_hash.to_json + end + end +end diff --git a/vendor/travis-core/lib/travis/settings/model_extensions.rb b/vendor/travis-core/lib/travis/settings/model_extensions.rb new file mode 100644 index 00000000..46f19a47 --- /dev/null +++ b/vendor/travis-core/lib/travis/settings/model_extensions.rb @@ -0,0 +1,177 @@ +module Travis + class Settings + module AccessorExtensions + def set(instance, value) + if instance.frozen? + raise 'setting values is not supported on default settings' + end + + super + end + + def get(instance) + if type.primitive <= Travis::Settings::EncryptedValue + unless instance.instance_variable_get(instance_variable_name) + value = Travis::Settings::EncryptedValue.new(nil) + if instance.frozen? + return value + else + set(instance, value) + end + end + end + + super instance + end + end + + module ModelExtensions + class Errors < ActiveModel::Errors + # Default behavior of Errors in Active Model is to + # translate symbolized message into full text message, + # using i18n if available. I don't want such a behavior, + # as I want to return error "codes" like :blank, not + # full text like "can't be blank" + def normalize_message(attribute, message, options) + message || :invalid + end + end + + module ClassMethods + def attribute(name, type = nil, options = {}) + options[:finalize] = false + + super name, type, options + + attribute = attribute_set[name] + attribute.extend(AccessorExtensions) + attribute.finalize + attribute.define_accessor_methods(attribute_set) + + self + end + + def load(json, additional_attributes = {}) + instance = new() + + json = JSON.parse(json) if json.is_a?(String) + instance.load json, additional_attributes + instance + end + end + + def self.included(base) + base.extend ClassMethods + end + + def additional_attributes + @additional_attributes || {} + end + + def additional_attributes=(hash = {}) + attribute_set.each do |attribute| + value = get(attribute.name) + if value.respond_to?(:additional_attributes=) + value.additional_attributes = hash + end + end + @additional_attributes = hash + end + + def errors + @errors ||= Errors.new(self) + end + + def attribute?(key) + attributes.keys.include? key.to_sym + end + + def to_hash + attributes.each_with_object({}) do |(name, value), hash| + hash[name] = value.respond_to?(:to_hash) ? value.to_hash : value + end + end + + def collection?(name) + # TODO: I don't like this type of class checking, it will be better to work + # based on an API contract, but it should do for now + if attribute = attribute_set[name.to_sym] + attribute.type.primitive <= Travis::Settings::Collection + end + end + + def encrypted?(name) + if attribute = attribute_set[name.to_sym] + attribute.type.primitive <= Travis::Settings::EncryptedValue + end + end + + def model?(name) + if attribute = attribute_set[name.to_sym] + attribute.type.primitive <= Travis::Settings::Model + end + end + + def primitive(name) + attribute_set[name.to_sym].type.primitive + end + + def get(key) + if attribute?(key) + self.send(key) + end + end + + def set(key, value) + if attribute?(key) + self.send("#{key}=", value) + end + end + + def simple_attributes + attributes.select { |k, v| simple_attribute?(k) } + end + + def simple_attribute?(key) + !(collection?(key) || encrypted?(key) || model?(key)) + end + + def load(hash = {}, additional_attributes = {}) + hash ||= {} + self.additional_attributes = additional_attributes || {} + + hash.merge(self.additional_attributes).each do |key, value| + if collection?(key) || encrypted?(key) || model?(key) + thing = get(key) + thing = set(key, primitive(key).new) if !thing && value + thing.load(value, self.additional_attributes) if thing + elsif attribute?(key) + set(key, value) + end + end + end + + def create(key, attributes) + attributes = (attributes || {}).merge(additional_attributes || {}) + set(key, primitive(key).new(attributes)) + get(key) + end + + def delete(key) + model = get(key) + set(key, nil) + model + end + + def update(key, attributes) + attributes = (attributes || {}).merge(additional_attributes || {}) + if model = get(key) + model.update(attributes) + model + else + create(key, attributes) + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/states_cache.rb b/vendor/travis-core/lib/travis/states_cache.rb new file mode 100644 index 00000000..8703b486 --- /dev/null +++ b/vendor/travis-core/lib/travis/states_cache.rb @@ -0,0 +1,158 @@ +require 'dalli' +require 'connection_pool' +require 'active_support/core_ext/module/delegation' +require 'travis/api' + +module Travis + class StatesCache + class CacheError < StandardError; end + + include Travis::Api::Formats + + attr_reader :adapter + + delegate :fetch, :to => :adapter + + def initialize(options = {}) + @adapter = options[:adapter] || MemcachedAdapter.new + end + + def write(id, branch, data) + if data.respond_to?(:id) + data = { + 'id' => data.id, + 'state' => data.state.to_s + } + end + + adapter.write(id, branch, data) + end + + def fetch_state(id, branch) + data = fetch(id, branch) + data['state'].to_sym if data && data['state'] + end + + class TestAdapter + attr_reader :calls + def initialize + @calls = [] + end + + def fetch(id, branch) + calls << [:fetch, id, branch] + end + + def write(id, branch, data) + calls << [:write, id, branch, data] + end + + def clear + calls.clear + end + end + + class MemcachedAdapter + attr_reader :pool + attr_accessor :jitter + attr_accessor :ttl + + def initialize(options = {}) + @pool = ConnectionPool.new(:size => 10, :timeout => 3) do + if options[:client] + options[:client] + else + new_dalli_connection + end + end + @jitter = 0.5 + @ttl = 7.days + end + + def fetch(id, branch = nil) + data = get(key(id, branch)) + data ? JSON.parse(data) : nil + end + + def write(id, branch, data) + build_id = data['id'] + data = data.to_json + + Travis.logger.info("[states-cache] Writing states cache for repo_id=#{id} branch=#{branch} build_id=#{build_id}") + set(key(id), data) if update?(id, nil, build_id) + set(key(id, branch), data) if update?(id, branch, build_id) + end + + def update?(id, branch, build_id) + current_data = fetch(id, branch) + return true unless current_data + + current_id = current_data['id'].to_i + new_id = build_id.to_i + + update = new_id >= current_id + message = "[states-cache] Checking if cache is stale for repo_id=#{id} branch=#{branch}. " + if update + message << "The cache is going to get an update, " + else + message << "The cache is fresh, " + end + message << "last cached build id=#{current_id}, we're checking build with id=#{new_id}" + Travis.logger.info(message) + + return update + end + + def key(id, branch = nil) + key = "state:#{id}" + if branch + key << "-#{branch}" + end + key + end + + private + + def new_dalli_connection + Dalli::Client.new(Travis.config.states_cache.memcached_servers, Travis.config.states_cache.memcached_options) + end + + def get(key) + retry_ringerror do + pool.with { |client| client.get(key) } + end + rescue Dalli::RingError => e + Metriks.meter("memcached.connect-errors").mark + raise CacheError, "Couldn't connect to a memcached server: #{e.message}" + end + + def set(key, data) + retry_ringerror do + pool.with { |client| client.set(key, data) } + Travis.logger.info("[states-cache] Setting cache for key=#{key} data=#{data}") + end + rescue Dalli::RingError => e + Metriks.meter("memcached.connect-errors").mark + Travis.logger.info("[states-cache] Writing cache key failed key=#{key} data=#{data}") + raise CacheError, "Couldn't connect to a memcached server: #{e.message}" + end + + def retry_ringerror + retries = 0 + begin + yield + rescue Dalli::RingError + retries += 1 + if retries <= 3 + # Sleep for up to 1/2 * (2^retries - 1) seconds + # For retries <= 3, this means up to 3.5 seconds + sleep(jitter * (rand(2 ** retries - 1) + 1)) + retry + else + raise + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/task.rb b/vendor/travis-core/lib/travis/task.rb new file mode 100644 index 00000000..dad3b98f --- /dev/null +++ b/vendor/travis-core/lib/travis/task.rb @@ -0,0 +1,93 @@ +require 'faraday' +require 'core_ext/hash/compact' +require 'core_ext/hash/deep_symbolize_keys' +require 'active_support/core_ext/string' +require 'active_support/core_ext/class/attribute' + +module Travis + class Task + include Logging + extend Instrumentation + + class_attribute :run_local + + class << self + extend Exceptions::Handling + + def run(queue, *args) + Travis::Async.run(self, :perform, async_options(queue), *args) + end + + def run_local? + !!run_local || Travis::Features.feature_deactivated?(:travis_tasks) + end + + def inline_or_sidekiq + run_local? ? :inline : :sidekiq + end + + def async_options(queue) + { queue: queue, use: inline_or_sidekiq, retries: 8, backtrace: true } + end + + def perform(*args) + new(*args).run + end + end + + attr_reader :payload, :params + + def initialize(payload, params = {}) + @payload = payload.deep_symbolize_keys + @params = params.deep_symbolize_keys + end + + def run + timeout after: params[:timeout] || 60 do + process + end + end + instrument :run + + private + + def repository + @repository ||= payload[:repository] + end + + def job + @job ||= payload[:job] + end + + def build + @build ||= payload[:build] + end + + def request + @request ||= payload[:request] + end + + def commit + @commit ||= payload[:commit] + end + + def pull_request? + build[:pull_request] + end + + def http + @http ||= Faraday.new(http_options) do |f| + f.request :url_encoded + f.adapter :net_http + end + end + + def http_options + { ssl: Travis.config.ssl.compact } + end + + def timeout(options = { after: 60 }, &block) + Timeout::timeout(options[:after], &block) + end + end +end diff --git a/vendor/travis-core/lib/travis/testing.rb b/vendor/travis-core/lib/travis/testing.rb new file mode 100644 index 00000000..445795fd --- /dev/null +++ b/vendor/travis-core/lib/travis/testing.rb @@ -0,0 +1,9 @@ +require 'faraday' +require 'core_ext/hash/compact' + +module Travis + module Testing + require 'travis/testing/stubs' + end +end + diff --git a/vendor/travis-core/lib/travis/testing/factories.rb b/vendor/travis-core/lib/travis/testing/factories.rb new file mode 100644 index 00000000..5490e84b --- /dev/null +++ b/vendor/travis-core/lib/travis/testing/factories.rb @@ -0,0 +1,135 @@ +require 'factory_girl' + +FactoryGirl.define do + factory :build do + owner { User.first || Factory(:user) } + repository { Repository.first || Factory(:repository) } + association :request + association :commit + started_at { Time.now.utc } + finished_at { Time.now.utc } + number 1 + state :passed + end + + factory :commit do + commit '62aae5f70ceee39123ef' + branch 'master' + message 'the commit message' + committed_at '2011-11-11T11:11:11Z' + committer_name 'Sven Fuchs' + committer_email 'svenfuchs@artweb-design.de' + author_name 'Sven Fuchs' + author_email 'svenfuchs@artweb-design.de' + compare_url 'https://github.com/svenfuchs/minimal/compare/master...develop' + end + + factory :test, :class => 'Job::Test' do + owner { User.first || Factory(:user) } + repository { Repository.first || Factory(:repository) } + commit { Factory(:commit) } + source { Factory(:build) } + log { Factory(:log) } + config { { 'rvm' => '1.8.7', 'gemfile' => 'test/Gemfile.rails-2.3.x' } } + number '2.1' + tags "" + end + + factory :log do + content '$ bundle install --pa' + end + + factory :request do + repository { Repository.first || Factory(:repository) } + association :commit + token 'the-token' + event_type 'push' + end + + factory :repository do + owner { User.find_by_login('svenfuchs') || Factory(:user) } + name 'minimal' + owner_name 'svenfuchs' + owner_email 'svenfuchs@artweb-design.de' + active true + url { |r| "http://github.com/#{r.owner_name}/#{r.name}" } + created_at { |r| Time.utc(2011, 01, 30, 5, 25) } + updated_at { |r| r.created_at + 5.minutes } + last_build_state :passed + last_build_number '2' + last_build_id 2 + last_build_started_at { Time.now.utc } + last_build_finished_at { Time.now.utc } + sequence(:github_id) {|n| n } + end + + factory :minimal, :parent => :repository do + end + + factory :enginex, :parent => :repository do + name 'enginex' + owner_name 'josevalim' + owner_email 'josevalim@email.com' + owner { User.find_by_login('josevalim') || Factory(:user, :login => 'josevalim') } + end + + factory :event do + repository { Repository.first || Factory(:repository) } + source { Build.first || Factory(:build) } + event 'build:started' + end + + factory :user do + name 'Sven Fuchs' + login 'svenfuchs' + email 'sven@fuchs.com' + tokens { [Token.new] } + github_oauth_token 'github_oauth_token' + end + + factory :org, :class => 'Organization' do + name 'travis-ci' + end + + factory :running_build, :parent => :build do + repository { Factory(:repository, :name => 'running_build') } + state :started + end + + factory :successful_build, :parent => :build do + repository { |b| Factory(:repository, :name => 'successful_build') } + state :passed + started_at { Time.now.utc } + finished_at { Time.now.utc } + end + + factory :broken_build, :parent => :build do + repository { Factory(:repository, :name => 'broken_build', :last_build_state => :failed) } + state :failed + started_at { Time.now.utc } + finished_at { Time.now.utc } + end + + factory :broken_build_with_tags, :parent => :build do + repository { Factory(:repository, :name => 'broken_build_with_tags', :last_build_state => :errored) } + matrix {[Factory(:test, :tags => "database_missing,rake_not_bundled", :number => "1.1"), + Factory(:test, :tags => "database_missing,log_limit_exceeded", :number => "1.2")]} + state :failed + started_at { Time.now.utc } + finished_at { Time.now.utc } + end + + factory :annotation do + url "https://travis-ci.org/travis-ci/travis-ci/jobs/12345" + description "Job passed" + job { Factory(:test) } + annotation_provider { Factory(:annotation_provider) } + end + + factory :annotation_provider do + name "Travis CI" + api_username "travis-ci" + api_key "0123456789abcdef" + end +end + diff --git a/vendor/travis-core/lib/travis/testing/matchers.rb b/vendor/travis-core/lib/travis/testing/matchers.rb new file mode 100644 index 00000000..d768d54c --- /dev/null +++ b/vendor/travis-core/lib/travis/testing/matchers.rb @@ -0,0 +1,50 @@ +# RSpec::Matchers.define :serve_result_image do |result| +# match do |request| +# path = "#{Rails.root}/public/images/result/#{result}.png" +# controller.expects(:send_file).with(path, { :type => 'image/png', :disposition => 'inline' }).once +# request.call +# end +# end + +RSpec::Matchers.define :issue_queries do |count| + match do |code| + queries = call(code) + + failure_message_for_should do + (["expected #{count} queries to be issued, but got #{queries.size}:"] + queries).join("\n\n") + end + + queries.size == count + end + + def call(code) + queries = [] + ActiveSupport::Notifications.subscribe 'sql.active_record' do |name, start, finish, id, payload| + queries << payload[:sql] unless payload[:sql] =~ /(?:ROLLBACK|pg_|BEGIN|COMMIT)/ + end + code.call + queries + end +end + +RSpec::Matchers.define :publish_instrumentation_event do |data| + match do |event| + non_matching = data.map { |key, value| [key, value, event[key]] unless event[key] == value }.compact + expected_keys = [:uuid, :event, :started_at] + missing_keys = expected_keys.select { |key| !event.key?(key) } + + failure_message_for_should do + message = "Expected a notification event to be published:\n\n\t#{event.inspect}\n\n" + message << "Including:\n\n\t#{data.inspect}\n\n" + + non_matching.each do |key, expected, actual| + message << "#{key.inspect} expected to be\n\n\t#{expected.inspect}\n\nbut was\n\n\t#{actual.inspect}\n\n" + end + + message << "Expected #{missing_keys.map(&:inspect).join(', ')} to be present." if missing_keys.present? + message + end + + non_matching.empty? && missing_keys.empty? + end +end diff --git a/vendor/travis-core/lib/travis/testing/payloads.rb b/vendor/travis-core/lib/travis/testing/payloads.rb new file mode 100644 index 00000000..e2c42578 --- /dev/null +++ b/vendor/travis-core/lib/travis/testing/payloads.rb @@ -0,0 +1,248 @@ +PAYLOADS = {} + +PAYLOADS[:github] = { + "private-repo" => %({ + "repository": { + "url": "http://github.com/svenfuchs/gem-release", + "name": "gem-release", + "private":true, + "owner": { + "email": "svenfuchs@artweb-design.de", + "name": "svenfuchs" + } + }, + "commits": [{ + "id": "9854592", + "message": "Bump to 0.0.15", + "timestamp": "2010-10-27 04:32:37", + "committer": { + "name": "Sven Fuchs", + "email": "svenfuchs@artweb-design.de" + }, + "author": { + "name": "Christopher Floess", + "email": "chris@flooose.de" + } + }], + "ref": "refs/heads/master" + }), + + "gem-release" => %({ + "repository": { + "url": "http://github.com/svenfuchs/gem-release", + "name": "gem-release", + "owner": { + "email": "svenfuchs@artweb-design.de", + "name": "svenfuchs" + } + }, + "commits": [{ + "id": "46ebe012ef3c0be5542a2e2faafd48047127e4be", + "message": "Bump to 0.0.15", + "timestamp": "2010-10-27 04:32:37", + "committer": { + "name": "Sven Fuchs", + "email": "svenfuchs@artweb-design.de" + }, + "author": { + "name": "Christopher Floess", + "email": "chris@flooose.de" + } + }], + "ref": "refs/heads/master", + "compare": "https://github.com/svenfuchs/gem-release/compare/af674bd...9854592" + }), + + "gh-pages-update" => %({ + "repository": { + "url": "http://github.com/svenfuchs/gem-release", + "name": "gem-release", + "owner": { + "email": "svenfuchs@artweb-design.de", + "name": "svenfuchs" + } + }, + "commits": [{ + "id": "9854592", + "message": "Bump to 0.0.15", + "timestamp": "2010-10-27 04:32:37", + "committer": { + "name": "Sven Fuchs", + "email": "svenfuchs@artweb-design.de" + }, + "author": { + "name": "Christopher Floess", + "email": "chris@flooose.de" + } + }], + "ref": "refs/heads/gh-pages" + }), + + "gh_pages-update" => %({ + "repository": { + "url": "http://github.com/svenfuchs/gem-release", + "name": "gem-release", + "owner": { + "email": "svenfuchs@artweb-design.de", + "name": "svenfuchs" + } + }, + "commits": [{ + "id": "9854592", + "message": "Bump to 0.0.15", + "timestamp": "2010-10-27 04:32:37", + "committer": { + "name": "Sven Fuchs", + "email": "svenfuchs@artweb-design.de" + }, + "author": { + "name": "Christopher Floess", + "email": "chris@flooose.de" + } + }], + "ref": "refs/heads/gh_pages" + }), + + # it is unclear why this payload was send but it happened quite often. the force option + # seems to indicate something like $ git push --force + "force-no-commit" => %({ + "pusher": { "name": "LTe", "email":"lite.88@gmail.com" }, + "repository":{ + "name":"acts-as-messageable", + "created_at":"2010/08/02 07:41:30 -0700", + "has_wiki":true, + "size":200, + "private":false, + "watchers":13, + "fork":false, + "url":"https://github.com/LTe/acts-as-messageable", + "language":"Ruby", + "pushed_at":"2011/05/31 04:16:01 -0700", + "open_issues":0, + "has_downloads":true, + "homepage":"http://github.com/LTe/acts-as-messageable", + "has_issues":true, + "forks":5, + "description":"ActsAsMessageable", + "owner": { "name":"LTe", "email":"lite.88@gmail.com" } + }, + "ref_name":"v0.3.0", + "forced":true, + "after":"b842078c2f0084bb36cea76da3dad09129b3c26b", + "deleted":false, + "ref":"refs/tags/v0.3.0", + "commits":[], + "base_ref":"refs/heads/master", + "before":"0000000000000000000000000000000000000000", + "compare":"https://github.com/LTe/acts-as-messageable/compare/v0.3.0", + "created":true + }), + + + :hook_inactive => %({ + "last_response": { + "status": "ok", + "message": "", + "code": 200 + }, + "config": { + "domain": "staging.travis-ci.org", + "user": "svenfuchs", + "token": "token" + }, + "created_at": "2011-09-18T10:49:06Z", + "events": [ + "push", + "pull_request", + "issue_comment", + "public", + "member" + ], + "active": false, + "updated_at": "2012-08-09T09:32:42Z", + "name": "travis", + "_links": { + "self": { + "href": "https://api.github.com/repos/svenfuchs/minimal/hooks/77103" + } + }, + "id": 77103 + }), + + :hook_active => %({ + "last_response": { + "status": "ok", + "message": "", + "code": 200 + }, + "config": { + "domain": "staging.travis-ci.org", + "user": "svenfuchs", + "token": "token" + }, + "created_at": "2011-09-18T10:49:06Z", + "events": [ + "push", + "pull_request", + "issue_comment", + "public", + "member" + ], + "active": true, + "updated_at": "2012-08-09T09:32:42Z", + "name": "travis", + "_links": { + "self": { + "href": "https://api.github.com/repos/svenfuchs/minimal/hooks/77103" + } + }, + "id": 77103 + }), + + :oauth => { + "uid" => "234423", + "user_info" => { + "name" => "John", + "nickname" => "john", + "email" => "john@email.com" + }, + "credentials" => { + "token" => "1234567890abcdefg" + } + }, + + :oauth_updated => { + "uid" => "234423", + "user_info" => { + "name" => "Johnathan", + "nickname" => "johnathan", + "email" => "johnathan@email.com" + }, + "credentials" => { + "token" => "1234567890abcdefg" + } + } +} + +PAYLOADS[:worker] = { + 'job:test:started' => { 'id' => 1, 'state' => 'started', 'started_at' => '2011-01-01 00:02:00 +0200', 'worker' => 'ruby3.worker.travis-ci.org:travis-ruby-4' }, + 'job:test:log' => { 'id' => 1, 'log' => '... appended' }, + 'job:test:log:1' => { 'id' => 1, 'log' => 'the ' }, + 'job:test:log:2' => { 'id' => 1, 'log' => 'full ' }, + 'job:test:log:3' => { 'id' => 1, 'log' => 'log' }, + 'job:test:finished' => { 'id' => 1, 'state' => 'finished', 'finished_at' => '2011-01-01 00:03:00 +0200', 'result' => 0, 'log' => 'the full log' } +} + +PAYLOADS[:queue] = { + 'job:test:1' => { + :build => { :id => 2, :number => '1.1', :commit => '9854592', :branch => 'master', :config => { :rvm => '1.8.7' } }, + :repository => { :id => 1, :slug => 'svenfuchs/gem-release' }, + :queue => 'builds' + }, + 'job:test:2' => { + :build => { :id => 3, :number => '1.2', :commit => '9854592', :branch => 'master', :config => { :rvm => '1.9.2' } }, + :repository => { :id => 1, :slug => 'svenfuchs/gem-release' }, + :queue => 'builds' + } +} + diff --git a/vendor/travis-core/lib/travis/testing/scenario.rb b/vendor/travis-core/lib/travis/testing/scenario.rb new file mode 100644 index 00000000..6a2daa78 --- /dev/null +++ b/vendor/travis-core/lib/travis/testing/scenario.rb @@ -0,0 +1,115 @@ +module Scenario + class << self + def default + minimal, enginex = repositories :minimal, :enginex + + build :repository => minimal, + :owner => minimal.owner, + :number => 1, + :config => { 'rvm' => ['1.8.7', '1.9.2'], 'gemfile' => ['test/Gemfile.rails-2.3.x', 'test/Gemfile.rails-3.0.x'] }, + :state => 'failed', + :started_at => '2010-11-12 12:00:00', + :finished_at => '2010-11-12 12:00:10', + :commit => { + :commit => '1a738d9d6f297c105ae2', + :ref => 'refs/heads/develop', + :branch => 'master', + :message => 'add Gemfile', + :committer_name => 'Sven Fuchs', + :committer_email => 'svenfuchs@artweb-design.de', + :committed_at => '2010-11-12 11:50:00', + }, + :jobs => [ + { :owner => minimal.owner, :log => Log.new(:content => 'minimal log 1'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' } + ] + + build :repository => minimal, + :owner => minimal.owner, + :number => 2, + :config => { 'rvm' => ['1.8.7', '1.9.2'], 'gemfile' => ['test/Gemfile.rails-2.3.x', 'test/Gemfile.rails-3.0.x'] }, + :state => 'passed', + :started_at => '2010-11-12 12:30:00', + :finished_at => '2010-11-12 12:30:20', + :commit => { + :commit => '91d1b7b2a310131fe3f8', + :ref => 'refs/heads/master', + :branch => 'master', + :message => 'Bump to 0.0.22', + :committer_name => 'Sven Fuchs', + :committer_email => 'svenfuchs@artweb-design.de', + :committed_at => '2010-11-12 12:25:00', + }, + :jobs => [ + { :owner => minimal.owner, :log => Log.new(:content => 'minimal log 2'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' } + ] + + build :repository => minimal, + :owner => minimal.owner, + :number => '3', + :config => { 'rvm' => ['1.8.7', '1.9.2'], 'gemfile' => ['test/Gemfile.rails-2.3.x', 'test/Gemfile.rails-3.0.x'] }, + :state => 'configured', + :started_at => '2010-11-12 13:00:00', + :finished_at => nil, + :commit => { + :commit => 'add057e66c3e1d59ef1f', + :ref => 'refs/heads/master', + :branch => 'master', + :message => 'unignore Gemfile.lock', + :committed_at => '2010-11-12 12:55:00', + :committer_name => 'Sven Fuchs', + :committer_email => 'svenfuchs@artweb-design.de', + }, + :jobs => [ + { :owner => minimal.owner, :log => Log.new(:content => 'minimal log 3.1'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' }, + { :owner => minimal.owner, :log => Log.new(:content => 'minimal log 3.2'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' }, + { :owner => minimal.owner, :log => Log.new(:content => 'minimal log 3.3'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' }, + { :owner => minimal.owner, :log => Log.new(:content => 'minimal log 3.4'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' } + ] + + build :repository => enginex, + :owner => enginex.owner, + :number => 1, + :state => 'failes', + :started_at => '2010-11-11 12:00:00', + :finished_at => '2010-11-11 12:00:05', + :commit => { + :commit => '565294c05913cfc23230', + :branch => 'master', + :ref => 'refs/heads/master', + :message => 'Update Capybara', + :author_name => 'Jose Valim', + :author_email => 'jose@email.com', + :committer_name => 'Jose Valim', + :committer_email => 'jose@email.com', + :committed_at => '2010-11-11 11:55:00', + }, + :jobs => [ + { :owner => enginex.owner, :log => Log.new(:content => 'enginex log 1'), :worker => 'ruby3.worker.travis-ci.org:travis-ruby-4' } + ] + + [minimal, enginex] + end + + def repositories(*names) + names.map { |name| Factory(name) } + end + + def build(attributes) + commit = attributes.delete(:commit) + jobs = attributes.delete(:jobs) + commit = Factory(:commit, commit) + + build = Factory(:build, attributes.merge(:commit => commit)) + build.matrix.each_with_index do |job, ix| + job.update_attributes!(jobs[ix] || {}) + end + + if build.finished? + keys = %w(id number state finished_at started_at) + attributes = keys.inject({}) { |result, key| result.merge(:"last_build_#{key}" => build.send(key)) } + build.repository.update_attributes!(attributes) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/testing/stubs.rb b/vendor/travis-core/lib/travis/testing/stubs.rb new file mode 100644 index 00000000..f4b08611 --- /dev/null +++ b/vendor/travis-core/lib/travis/testing/stubs.rb @@ -0,0 +1,356 @@ +require 'active_support/core_ext/numeric/time' + +module Travis + module Testing + module Stubs + require 'travis/testing/stubs/stub' + + class << self + include Stub + + def included(base) + base.send(:instance_eval) do + let(:repository) { stub_repo } + let(:repo) { stub_repo } + let(:request) { stub_request } + let(:commit) { stub_commit } + let(:build) { stub_build } + let(:test) { stub_test } + let(:log) { stub_log } + let(:annotation) { stub_annotation } + let(:annotation_provider) { stub_annotation_provider } + let(:event) { stub_event } + let(:worker) { stub_worker } + let(:user) { stub_user } + let(:org) { stub_org } + let(:url) { stub_url } + let(:broadcast) { stub_broadcast } + let(:travis_token) { stub_travis_token } + let(:cache) { stub_cache } + end + end + end + + def stub_repo(attributes = {}) + Stubs.stub 'repository', attributes.reverse_merge( + id: 1, + owner: stub_user(id: 1, login: 'svenfuchs'), + owner_type: 'User', + owner_id: 1, + owner_name: 'svenfuchs', + owner_email: 'svenfuchs@artweb-design.de', + name: 'minimal', + slug: 'svenfuchs/minimal', + description: 'the repo description', + url: 'http://github.com/svenfuchs/minimal', + source_url: 'git://github.com/svenfuchs/minimal.git', + api_url: 'https://api.github.com/repos/svenfuchs/minimal', + key: stub_key, + admin: stub_user, + active: true, + private: false, + private?: false, + last_build_id: 1, + last_build_number: 2, + last_build_started_at: Time.now.utc - 60, + last_build_finished_at: Time.now.utc, + last_build_state: :passed, + last_build_duration: 60, + github_language: 'ruby', + github_id: 549743, + builds_only_with_travis_yml?: false, + settings: stub_settings, + users_with_permission: [], + default_branch: 'master', + current_build_id: nil + ) + end + alias stub_repository stub_repo + + def stub_settings + Stubs.stub 'settings', 'ssh_keys' => [], 'env_vars' => [] + end + + def stub_key(attributes = {}) + Stubs.stub 'key', attributes.reverse_merge( + id: 1, + public_key: '-----BEGIN PUBLIC KEY-----' + ) + end + + def stub_request(attributes = {}) + repo = stub_repository + commit = stub_commit + Stubs.stub 'request', attributes.reverse_merge( + id: 1, + repository: repo, + repository_id: repo.id, + commit: commit, + commit_id: commit.id, + config: {}, + event_type: 'push', + head_commit: 'head-commit', + base_commit: 'base-commit', + token: 'token', + api_request?: false, + pull_request?: false, + comments_url: 'http://github.com/path/to/comments', + config_url: 'https://api.github.com/repos/svenfuchs/minimal/contents/.travis.yml?ref=62aae5f70ceee39123ef', + result: :accepted, + created_at: DateTime.new(2013, 01, 01, 0, 0, 0), + owner_type: 'User', + owner_id: 1, + owner_name: 'svenfuchs', + owner_email: 'svenfuchs@artweb-design.de', + message: 'a message', + branch_name: 'master', + tag_name: '', + pull_request: false, + pull_request_title: nil, + pull_request_number: nil, + head_repo: 'BanzaiMan/travis-core', + head_branch: 'feature-branch', + base_repo: 'travis-ci/travis-core', + base_branch: 'master', + ) + end + + def stub_commit(attributes = {}) + Stubs.stub 'commit', attributes.reverse_merge( + id: 1, + commit: '62aae5f70ceee39123ef', + range: '0cd9ffaab2c4ffee...62aae5f70ceee39123ef', + branch: 'master', + ref: 'refs/master', + message: 'the commit message', + author_name: 'Sven Fuchs', + author_email: 'svenfuchs@artweb-design.de', + committer_name: 'Sven Fuchs', + committer_email: 'svenfuchs@artweb-design.de', + committed_at: Time.now.utc - 3600, + compare_url: 'https://github.com/svenfuchs/minimal/compare/master...develop', + pull_request?: false, + pull_request_number: nil + ) + end + + def stub_build(attributes = {}) + Stubs.stub 'build', attributes.reverse_merge( + id: 1, + repository_id: repository.id, + repository: stub_repository(owner: attributes.delete(:owner)), + request_id: request.id, + request: request, + commit_id: commit.id, + commit: commit, + matrix: attributes[:matrix] || [stub_test(id: 1, number: '2.1'), stub_test(id: 2, number: '2.2')], + matrix_ids: [1, 2], + cached_matrix_ids: [1, 2], + number: 2, + config: { 'rvm' => ['1.8.7', '1.9.2'], 'gemfile' => ['test/Gemfile.rails-2.3.x', 'test/Gemfile.rails-3.0.x'] }, + obfuscated_config: { 'rvm' => ['1.8.7', '1.9.2'], 'gemfile' => ['test/Gemfile.rails-2.3.x', 'test/Gemfile.rails-3.0.x'] }, + state: 'passed', + result: 0, # see build/compat.rb + passed?: true, + failed?: false, + finished?: true, + previous_state: 'passed', + started_at: Time.now.utc - 60, + finished_at: Time.now.utc, + duration: 60, + pull_request?: false, + queue: 'builds.linux', + pull_request_title: nil, + pull_request_number: nil, + secure_env_enabled?: true, + event_type: 'push' + ) + end + + def stub_test(attributes = {}) + log = self.log + annotation = stub_annotation(job_id: 1) + test = Stubs.stub 'test', attributes.reverse_merge( + id: 1, + owner: stub_user, + repository_id: 1, + repository: repository, + source_id: 1, + request_id: 1, + commit_id: commit.id, + commit: commit, + log: log, + log_id: log.id, + annotations: [annotation], + annotation_ids: [annotation.id], + number: '2.1', + config: { 'rvm' => '1.8.7', 'gemfile' => 'test/Gemfile.rails-2.3.x' }, + decrypted_config: { 'rvm' => '1.8.7', 'gemfile' => 'test/Gemfile.rails-2.3.x' }, + obfuscated_config: { 'rvm' => '1.8.7', 'gemfile' => 'test/Gemfile.rails-2.3.x' }, + state: :passed, + result: 0, # see job/compat.rb + started?: true, + finished?: true, + queue: 'builds.linux', + allow_failure: false, + started_at: Time.now.utc - 60, + finished_at: Time.now.utc, + worker: 'ruby3.worker.travis-ci.org:travis-ruby-4', + tags: 'tag-a,tag-b', + log_content: log.content, + ssh_key: nil, + secure_env_enabled?: true + ) + + source = stub_build(:matrix => [test]) + test.define_singleton_method(:source) { source } + test + end + + def stub_log(attributes = {}) + Stubs.stub 'log', attributes.reverse_merge( + class: Stubs.stub('class', name: 'Log'), + id: 1, + job_id: 1, + content: 'the test log' + ) + end + + def stub_log_part(attributes = {}) + Stubs.stub 'log_part', attributes.reverse_merge( + id: 1, + log_id: 1, + content: 'the test log', + number: 1, + final: false + ) + end + + def stub_annotation(attributes = {}) + Stubs.stub 'annotation', attributes.reverse_merge( + class: Stubs.stub('class', name: 'Annotation'), + id: 1, + job_id: attributes[:job_id] || test.id, # Needed to break the infinite loop in stub_test + annotation_provider_id: annotation_provider.id, + annotation_provider: annotation_provider, + description: "The job passed.", + url: "https://travis-ci.org/travis-ci/travis-ci/12345", + status: "" + ) + end + + def stub_annotation_provider(attributes = {}) + Stubs.stub 'annotation_provider', attributes.reverse_merge( + class: Stubs.stub('class', name: 'AnnotationProvider'), + id: 1, + name: "Travis CI", + api_username: "travis-ci", + api_key: "0123456789abcdef", + ) + end + + def stub_event(attributes = {}) + Stubs.stub 'event', attributes.reverse_merge( + id: 1, + repository_id: 1, + source: stub_request, + source_id: stub_request.id, + source_type: 'Request', + event: 'request:finished', + data: { 'result' => 'accepted' }, + created_at: Time.now + ) + end + + def stub_worker(attributes = {}) + Stubs.stub 'worker', attributes.reverse_merge( + id: 1, + name: 'ruby-1', + host: 'ruby-1.worker.travis-ci.org', + queue: 'builds.linux', + state: 'created', + last_seen_at: Time.now.utc, + payload: nil, + ) + end + + def stub_user(attributes = {}) + Stubs.stub 'user', attributes.reverse_merge( + id: 1, + github_id: 1, + organizations: [org], + name: 'Sven Fuchs', + login: 'svenfuchs', + email: 'svenfuchs@artweb-design.de', + gravatar_id: '402602a60e500e85f2f5dc1ff3648ecb', + avatar_url: 'https://0.gravatar.com/avatar/402602a60e500e85f2f5dc1ff3648ecb', + locale: 'de', + github_oauth_token: 'token', + syncing?: false, + is_syncing: false, + synced_at: Time.now.utc - 3600, + tokens: [stub('token', token: 'token')], + github_scopes: Travis.config.oauth2.to_h[:scopes].to_s.split(','), + correct_scopes?: true, + created_at: Time.now.utc - 7200 + ) + end + + def stub_org(attributes = {}) + Stubs.stub 'org', attributes.reverse_merge( + id: 1, + login: 'travis-ci', + name: 'Travis CI', + email: 'contact@travis-ci.org' + ) + end + + def stub_url(attributes = {}) + Stubs.stub 'url', attributes.reverse_merge( + id: 1, + short_url: 'http://trvs.io/short' + ) + end + + def stub_broadcast(attributes = {}) + Stubs.stub 'broadcast', attributes.reverse_merge( + id: 1, + message: 'message' + ) + end + + def stub_travis_token(attributes = {}) + Stubs.stub 'travis_token', attributes.reverse_merge( + id: 1, + user: stub_user, + token: 'super secret' + ) + end + + def stub_cache(attributes = {}) + Stubs.stub 'cache', attributes.reverse_merge( + repository: stub_repository, + size: 1000, + slug: 'cache', + branch: 'master', + last_modified: Time.at(0).utc + ) + end + + def stub_email(attributes = {}) + Stubs.stub 'email', attributes.reverse_merge( + email: 'email' + ) + end + + def stub_job(attributes = {}) + Stubs.stub 'job', attributes.reverse_merge( + repository: stub_repository, + id: '42.1', + enqueue: true + ) + end + end + end +end + diff --git a/vendor/travis-core/lib/travis/testing/stubs/stub.rb b/vendor/travis-core/lib/travis/testing/stubs/stub.rb new file mode 100644 index 00000000..fcad2c9e --- /dev/null +++ b/vendor/travis-core/lib/travis/testing/stubs/stub.rb @@ -0,0 +1,45 @@ +module Travis + module Testing + module Stubs + module Stub + def stub(name, attributes) + Object.new.tap do |object| + meta_class = (class << object; self; end) + class_stub = stub_class(name.camelize) + + attributes.each do |name, value| + meta_class.send(:define_method, name) { |*| value } + end + + meta_class.send(:define_method, :class) do + class_stub + end + + meta_class.send(:define_method, :is_a?) do |const| + const.name.to_s == name.to_s.camelize + end + + meta_class.send(:define_method, :inspect) do + attrs = attributes.map { |name, value| [name, value.inspect].join('=') }.join(' ') + "#<#{name.camelize}:#{object.object_id} #{attrs}>" + end + end + end + + # TODO needs to take care of nested namespaces, so we can pass 'job/test' + def stub_class(name) + if const_defined?(*method(:const_defined?).arity == 1 ? [name] : [name, false]) + const_get(name) + else + Class.new.tap do |const| + const_set(name, const) + meta_class = (class << const; self; end) + meta_class.send(:define_method, :name) { name } + meta_class.send(:define_method, :inspect) { name } + end + end + end + end + end + end +end diff --git a/vendor/travis-core/lib/travis/travis_yml_stats.rb b/vendor/travis-core/lib/travis/travis_yml_stats.rb new file mode 100644 index 00000000..776e9318 --- /dev/null +++ b/vendor/travis-core/lib/travis/travis_yml_stats.rb @@ -0,0 +1,195 @@ +require "sidekiq" + +begin + require "keen" +rescue LoadError +end + +module Travis + class TravisYmlStats + class KeenPublisher + include ::Sidekiq::Worker + + sidekiq_options queue: :keen_events + + def perform(payload, deployment_payload = nil, notification_payload = nil) + if defined?(Keen) && ENV["KEEN_PROJECT_ID"] + payload = { :requests => [payload] } + payload[:deployments] = deployment_payload if deployment_payload.to_a.size > 0 + payload[:notifications] = notification_payload if notification_payload.to_a.size > 0 + Keen.publish_batch(payload) + end + end + end + + LANGUAGE_VERSION_KEYS = %w[ + ghc + go + jdk + node_js + otp_release + perl + php + python + ruby + rvm + scala + ] + + def self.store_stats(request, publisher=KeenPublisher) + new(request, publisher).store_stats + end + + def initialize(request, publisher) + @request = request + @publisher = publisher + @keen_payload = {} + @keen_payload_deployment = [] + @keen_payload_notification = [] + end + + def store_stats + set_basic_info + set_language + set_language_version + set_uses_sudo + set_uses_apt_get + set_dist + set_group + set_deployment_provider_count + set_notification + + @publisher.perform_async(keen_payload, keen_payload_deployment, keen_payload_notification) + end + + private + + attr_reader :request, :keen_payload + attr_accessor :keen_payload_deployment + attr_accessor :keen_payload_notification + + def set(path, value, collection = keen_payload) + path = Array(path) + hsh = collection + path[0..-2].each do |key| + hsh[key.to_sym] ||= {} + hsh = hsh[key.to_sym] + end + + hsh[path.last.to_sym] = value + end + + def set_basic_info + set :event_type, request.event_type + set :matrix_size, request.builds.map { |build| build.matrix.size }.reduce(:+) + set :repository_id, request.repository_id + set :owner_id, request.owner_id + set :owner_type, request.owner_type + # The owner_type, owner_id tuple is there so we can do unique counts on it + set :owner, [request.owner_type, request.owner_id] + end + + def set_language + set :language, travis_yml_language + set :github_language, github_language + end + + def set_language_version + LANGUAGE_VERSION_KEYS.each do |key| + if config.key?(key) + case config[key] + when String, Array + set [:language_version, key], Array(config[key]).map(&:to_s).sort + else + set [:language_version, key], ["invalid"] + end + end + end + end + + def set_uses_sudo + set :uses_sudo, commands.any? { |command| command =~ /\bsudo\b/ } + end + + def set_uses_apt_get + set :uses_apt_get, commands.any? { |command| command =~ /\bapt-get\b/ } + end + + def set_dist + set :dist_name, dist_name + end + + def set_group + set :group_name, group_name + end + + def set_deployment_provider_count + deploy = config["deploy"] || return + # Hash#to_a is not what we want here + deployments = deploy.is_a?(Hash) ? [deploy] : Array(deploy) + deployments.map {|d| d["provider"] }.uniq.each do |provider| + keen_payload_deployment << { provider: provider.downcase, repository_id: request.repository_id } + end + rescue + nil + end + + def set_notification + notifications = config["notifications"] || return + notifications.keys.each do |notifier| + keen_payload_notification << { notifier: notifier.downcase, repository_id: request.repository_id } + end + rescue + nil + end + + def config + request.config + end + + def payload + request.payload.is_a?(String) ? MultiJson.decode(request.payload) : request.payload + end + + def commands + [ + config["before_install"], + config["install"], + config["before_script"], + config["script"], + config["after_success"], + config["after_failure"], + config["before_deploy"], + config["after_deploy"], + ].flatten.compact + end + + def travis_yml_language + language = config["language"] + case language + when String + language + when nil + "default" + else + "invalid" + end + end + + def github_language + payload.fetch("repository", {})["language"] + end + + def normalize_string(str) + str.downcase.gsub("#", "-sharp").gsub(/[^A-Za-z0-9.:\-_]/, "") + end + + def dist_name + config.fetch('dist', 'default') + end + + def group_name + config.fetch('group', 'default') + end + end +end diff --git a/vendor/travis-core/lib/travis_core/version.rb b/vendor/travis-core/lib/travis_core/version.rb new file mode 100644 index 00000000..f8fc62f6 --- /dev/null +++ b/vendor/travis-core/lib/travis_core/version.rb @@ -0,0 +1,3 @@ +module TravisCore + VERSION = "0.0.1" +end diff --git a/vendor/travis-core/travis-core.gemspec b/vendor/travis-core/travis-core.gemspec new file mode 100644 index 00000000..d34586ec --- /dev/null +++ b/vendor/travis-core/travis-core.gemspec @@ -0,0 +1,49 @@ +# encoding: utf-8 + +$:.unshift File.expand_path('../lib', __FILE__) +require 'travis_core/version' + +Gem::Specification.new do |s| + s.name = "travis-core" + s.version = TravisCore::VERSION + s.authors = ["Travis CI"] + s.email = "contact@travis-ci.org" + s.homepage = "https://github.com/travis-ci/travis-core" + s.summary = "The heart of Travis" + + s.files = Dir['{lib/**/*,spec/**/*,[A-Z]*}'] + s.platform = Gem::Platform::RUBY + s.require_path = 'lib' + s.rubyforge_project = '[none]' + + s.add_dependency 'rake' + s.add_dependency 'thor' + s.add_dependency 'activerecord', '~> 3.2.19' + s.add_dependency 'actionmailer', '~> 3.2.19' + s.add_dependency 'railties', '~> 3.2.19' + s.add_dependency 'rollout', '~> 1.1.0' + s.add_dependency 'coder', '~> 0.4.0' + s.add_dependency 'virtus', '~> 1.0.0' + + # travis + s.add_dependency 'travis-config', '~> 0.1.0' + + # db + s.add_dependency 'data_migrations', '~> 0.0.1' + s.add_dependency 'redis', '~> 3.0' + + + # structures + s.add_dependency 'hashr' + s.add_dependency 'metriks', '~> 0.9.7' + + # app + s.add_dependency 'simple_states', '~> 1.0.0' + + # apis + s.add_dependency 'pusher', '~> 0.14.0' + s.add_dependency 's3', '~> 0.3' + s.add_dependency 'gh' + s.add_dependency 'multi_json' + s.add_dependency 'google-api-client', '~> 0.9.4' +end