kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Merge branch 'stanhu/gitlab-ce-add-eager-load-lib' into 'master'

Browse source 
Add eager load paths to help prevent dependency load issues with Sidekiq workers

_Originally opened at !3545 by @stanhu._

- - -

Relevant resources:

- https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
- https://github.com/mperham/sidekiq/issues/1281#issuecomment-27129904
- http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
- 52ce6ece8c/railties/lib/rails/engine.rb (L472-L479)
- https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/paths.rb

Attempts to address #3661, #11896, #12769, #13521, #14131, #14589, #14759, #14825.

See merge request !3724
This commit is contained in:
Robert Speicher 2016-05-10 20:14:40 +00:00
commit 971662e6a4
11 changed files with 215 additions and 206 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG
View file

@ -32,6 +32,7 @@ v 8.8.0 (unreleased)
- API: Expose Issue#user_notes_count. !3126 (Anton Popov)
- Files over 5MB can only be viewed in their raw form, files over 1MB without highlighting !3718
- Add support for supressing text diffs using .gitattributes on the default branch (Matt Oakes)
- Add eager load paths to help prevent dependency load issues in Sidekiq workers. !3724
- Added multiple colors for labels in dropdowns when dups happen.
- Improve description for the Two-factor Authentication sign-in screen. (Connor Shea)
- API support for the 'since' and 'until' operators on commit requests (Paco Guzman)

2
app/models/repository.rb
View file

@ -81,7 +81,7 @@ class Repository
def commit(id = 'HEAD')
return nil unless exists?
commit = Gitlab::Git::Commit.find(raw_repository, id)
commit = Commit.new(commit, @project) if commit
commit = ::Commit.new(commit, @project) if commit
commit
rescue Rugged::OdbError
nil

25
config/application.rb
View file

@ -1,23 +1,30 @@
require File.expand_path('../boot', __FILE__)
require 'rails/all'
require 'devise'
I18n.config.enforce_available_locales = false
Bundler.require(:default, Rails.env)
require_relative '../lib/gitlab/redis'
module Gitlab
class Application < Rails::Application
require_dependency Rails.root.join('lib/gitlab/redis')
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
# -- all .rb files in that directory are automatically loaded.
# Custom directories with classes and modules you want to be autoloadable.
config.autoload_paths.push(*%W(#{config.root}/lib
#{config.root}/app/models/hooks
#{config.root}/app/models/concerns
#{config.root}/app/models/project_services
#{config.root}/app/models/members))
# Sidekiq uses eager loading, but directories not in the standard Rails
# directories must be added to the eager load paths:
# https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
# Also, there is no need to add `lib` to autoload_paths since autoloading is
# configured to check for eager loaded paths:
# https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/engine.rb#L687
# This is a nice reference article on autoloading/eager loading:
# http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
config.eager_load_paths.push(*%W(#{config.root}/lib
#{config.root}/app/models/ci
#{config.root}/app/models/hooks
#{config.root}/app/models/members
#{config.root}/app/models/project_services))
# Only load the plugins named here, in the order given (default is alphabetical).
# :all can be used as a placeholder for all plugins not explicitly named.

2
config/initializers/1_settings.rb
View file

@ -1,4 +1,4 @@
require 'gitlab' # Load lib/gitlab.rb as soon as possible
require_dependency Rails.root.join('lib/gitlab') # Load Gitlab as soon as possible
class Settings < Settingslogic
source ENV.fetch('GITLAB_CONFIG') { "#{Rails.root}/config/gitlab.yml" }

6
config/initializers/5_backend.rb
View file

@ -1,11 +1,11 @@
# GIT over HTTP
require Rails.root.join("lib", "gitlab", "backend", "grack_auth")
require_dependency Rails.root.join('lib/gitlab/backend/grack_auth')
# GIT over SSH
require Rails.root.join("lib", "gitlab", "backend", "shell")
require_dependency Rails.root.join('lib/gitlab/backend/shell')
# GitLab shell adapter
require Rails.root.join("lib", "gitlab", "backend", "shell_adapter")
require_dependency Rails.root.join('lib/gitlab/backend/shell_adapter')
required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)

67
lib/api/api.rb
View file

@ -1,5 +1,3 @@
Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file}
module API
class API < Grape::API
include APIGuard
@ -25,38 +23,39 @@ module API
format :json
content_type :txt, "text/plain"
helpers Helpers
# Ensure the namespace is right, otherwise we might load Grape::API::Helpers
helpers ::API::Helpers
mount Groups
mount GroupMembers
mount Users
mount Projects
mount Repositories
mount Issues
mount Milestones
mount Session
mount MergeRequests
mount Notes
mount Internal
mount SystemHooks
mount ProjectSnippets
mount ProjectMembers
mount DeployKeys
mount ProjectHooks
mount Services
mount Files
mount Commits
mount CommitStatus
mount Namespaces
mount Branches
mount Labels
mount Settings
mount Keys
mount Tags
mount Triggers
mount Builds
mount Variables
mount Runners
mount Licenses
mount ::API::Groups
mount ::API::GroupMembers
mount ::API::Users
mount ::API::Projects
mount ::API::Repositories
mount ::API::Issues
mount ::API::Milestones
mount ::API::Session
mount ::API::MergeRequests
mount ::API::Notes
mount ::API::Internal
mount ::API::SystemHooks
mount ::API::ProjectSnippets
mount ::API::ProjectMembers
mount ::API::DeployKeys
mount ::API::ProjectHooks
mount ::API::Services
mount ::API::Files
mount ::API::Commits
mount ::API::CommitStatuses
mount ::API::Namespaces
mount ::API::Branches
mount ::API::Labels
mount ::API::Settings
mount ::API::Keys
mount ::API::Tags
mount ::API::Triggers
mount ::API::Builds
mount ::API::Variables
mount ::API::Runners
mount ::API::Licenses
end
end

302
lib/api/api_guard.rb
View file

@ -2,171 +2,175 @@
require 'rack/oauth2'
module APIGuard
extend ActiveSupport::Concern
module API
module APIGuard
extend ActiveSupport::Concern
included do |base|
# OAuth2 Resource Server Authentication
use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
# The authenticator only fetches the raw token string
included do |base|
# OAuth2 Resource Server Authentication
use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
# The authenticator only fetches the raw token string
# Must yield access token to store it in the env
request.access_token
end
helpers HelperMethods
install_error_responders(base)
end
# Helper Methods for Grape Endpoint
module HelperMethods
# Invokes the doorkeeper guard.
#
# If token is presented and valid, then it sets @current_user.
#
# If the token does not have sufficient scopes to cover the requred scopes,
# then it raises InsufficientScopeError.
#
# If the token is expired, then it raises ExpiredError.
#
# If the token is revoked, then it raises RevokedError.
#
# If the token is not found (nil), then it raises TokenNotFoundError.
#
# Arguments:
#
# scopes: (optional) scopes required for this guard.
# Defaults to empty array.
#
def doorkeeper_guard!(scopes: [])
if (access_token = find_access_token).nil?
raise TokenNotFoundError
else
case validate_access_token(access_token, scopes)
when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
raise InsufficientScopeError.new(scopes)
when Oauth2::AccessTokenValidationService::EXPIRED
raise ExpiredError
when Oauth2::AccessTokenValidationService::REVOKED
raise RevokedError
when Oauth2::AccessTokenValidationService::VALID
@current_user = User.find(access_token.resource_owner_id)
end
# Must yield access token to store it in the env
request.access_token
end
helpers HelperMethods
install_error_responders(base)
end
def doorkeeper_guard(scopes: [])
if access_token = find_access_token
case validate_access_token(access_token, scopes)
when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
raise InsufficientScopeError.new(scopes)
# Helper Methods for Grape Endpoint
module HelperMethods
# Invokes the doorkeeper guard.
#
# If token is presented and valid, then it sets @current_user.
#
# If the token does not have sufficient scopes to cover the requred scopes,
# then it raises InsufficientScopeError.
#
# If the token is expired, then it raises ExpiredError.
#
# If the token is revoked, then it raises RevokedError.
#
# If the token is not found (nil), then it raises TokenNotFoundError.
#
# Arguments:
#
# scopes: (optional) scopes required for this guard.
# Defaults to empty array.
#
def doorkeeper_guard!(scopes: [])
if (access_token = find_access_token).nil?
raise TokenNotFoundError
when Oauth2::AccessTokenValidationService::EXPIRED
raise ExpiredError
when Oauth2::AccessTokenValidationService::REVOKED
raise RevokedError
when Oauth2::AccessTokenValidationService::VALID
@current_user = User.find(access_token.resource_owner_id)
end
end
end
def current_user
@current_user
end
private
def find_access_token
@access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
end
def doorkeeper_request
@doorkeeper_request ||= ActionDispatch::Request.new(env)
end
def validate_access_token(access_token, scopes)
Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
end
end
module ClassMethods
# Installs the doorkeeper guard on the whole Grape API endpoint.
#
# Arguments:
#
# scopes: (optional) scopes required for this guard.
# Defaults to empty array.
#
def guard_all!(scopes: [])
before do
guard! scopes: scopes
end
end
private
def install_error_responders(base)
error_classes = [ MissingTokenError, TokenNotFoundError,
ExpiredError, RevokedError, InsufficientScopeError]
base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
end
def oauth2_bearer_token_error_handler
Proc.new do |e|
response =
case e
when MissingTokenError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
when TokenNotFoundError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
:invalid_token,
"Bad Access Token.")
when ExpiredError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
:invalid_token,
"Token is expired. You can either do re-authorization or token refresh.")
when RevokedError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
:invalid_token,
"Token was revoked. You have to re-authorize from the user.")
when InsufficientScopeError
# FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
# does not include WWW-Authenticate header, which breaks the standard.
Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
:insufficient_scope,
Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
{ scope: e.scopes })
else
case validate_access_token(access_token, scopes)
when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
raise InsufficientScopeError.new(scopes)
when Oauth2::AccessTokenValidationService::EXPIRED
raise ExpiredError
when Oauth2::AccessTokenValidationService::REVOKED
raise RevokedError
when Oauth2::AccessTokenValidationService::VALID
@current_user = User.find(access_token.resource_owner_id)
end
end
end
response.finish
def doorkeeper_guard(scopes: [])
if access_token = find_access_token
case validate_access_token(access_token, scopes)
when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
raise InsufficientScopeError.new(scopes)
when Oauth2::AccessTokenValidationService::EXPIRED
raise ExpiredError
when Oauth2::AccessTokenValidationService::REVOKED
raise RevokedError
when Oauth2::AccessTokenValidationService::VALID
@current_user = User.find(access_token.resource_owner_id)
end
end
end
def current_user
@current_user
end
private
def find_access_token
@access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
end
def doorkeeper_request
@doorkeeper_request ||= ActionDispatch::Request.new(env)
end
def validate_access_token(access_token, scopes)
Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
end
end
end
#
# Exceptions
#
module ClassMethods
# Installs the doorkeeper guard on the whole Grape API endpoint.
#
# Arguments:
#
# scopes: (optional) scopes required for this guard.
# Defaults to empty array.
#
def guard_all!(scopes: [])
before do
guard! scopes: scopes
end
end
class MissingTokenError < StandardError; end
private
class TokenNotFoundError < StandardError; end
def install_error_responders(base)
error_classes = [ MissingTokenError, TokenNotFoundError,
ExpiredError, RevokedError, InsufficientScopeError]
class ExpiredError < StandardError; end
base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
end
class RevokedError < StandardError; end
def oauth2_bearer_token_error_handler
Proc.new do |e|
response =
case e
when MissingTokenError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
class InsufficientScopeError < StandardError
attr_reader :scopes
def initialize(scopes)
@scopes = scopes
when TokenNotFoundError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
:invalid_token,
"Bad Access Token.")
when ExpiredError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
:invalid_token,
"Token is expired. You can either do re-authorization or token refresh.")
when RevokedError
Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
:invalid_token,
"Token was revoked. You have to re-authorize from the user.")
when InsufficientScopeError
# FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
# does not include WWW-Authenticate header, which breaks the standard.
Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
:insufficient_scope,
Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
{ scope: e.scopes })
end
response.finish
end
end
end
#
# Exceptions
#
class MissingTokenError < StandardError; end
class TokenNotFoundError < StandardError; end
class ExpiredError < StandardError; end
class RevokedError < StandardError; end
class InsufficientScopeError < StandardError
attr_reader :scopes
def initialize(scopes)
@scopes = scopes
end
end
end
end

2
lib/api/commit_statuses.rb
View file

@ -2,7 +2,7 @@ require 'mime/types'
module API
# Project commit statuses API
class CommitStatus < Grape::API
class CommitStatuses < Grape::API
resource :projects do
before { authenticate! }

10
lib/ci/api/api.rb
View file

@ -1,9 +1,7 @@
Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file}
module Ci
module API
class API < Grape::API
include APIGuard
include ::API::APIGuard
version 'v1', using: :path
rescue_from ActiveRecord::RecordNotFound do
@ -31,9 +29,9 @@ module Ci
helpers ::API::Helpers
helpers Gitlab::CurrentSettings
mount Builds
mount Runners
mount Triggers
mount ::Ci::API::Builds
mount ::Ci::API::Runners
mount ::Ci::API::Triggers
end
end
end

2
lib/gitlab.rb
View file

@ -1,4 +1,4 @@
require 'gitlab/git'
require_dependency 'gitlab/git'
module Gitlab
def self.com?

2
spec/requests/api/commit_status_spec.rb → spec/requests/api/commit_statuses_spec.rb
View file

@ -1,6 +1,6 @@
require 'spec_helper'
describe API::CommitStatus, api: true do
describe API::CommitStatuses, api: true do
include ApiHelpers
let!(:project) { create(:project) }