From 6f292eaa69c771cec8a81d2edaad19a26ab3eae6 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 18 Apr 2018 15:41:42 +0200 Subject: [PATCH] Revert the addition of goldiloader This reverts the addition of the "goldiloader" Gem and all use of it. While this Gem is very promising it's causing a variety of problems on GitLab.com due to it eager-loading too much data in places where we don't expect/can handle this. At least for the time being this means we have to go back to manually fixing N+1 query problems, but at least those should not cause a negative impact on availability. --- Gemfile | 2 - Gemfile.lock | 4 - app/models/ci/runner.rb | 2 +- app/models/clusters/cluster.rb | 2 +- app/models/concerns/issuable.rb | 2 +- app/models/concerns/resolvable_discussion.rb | 2 +- app/models/deploy_key.rb | 2 +- app/models/deploy_token.rb | 2 +- app/models/fork_network.rb | 2 +- app/models/group.rb | 6 +- app/models/issue.rb | 2 +- app/models/label.rb | 4 +- app/models/lfs_object.rb | 2 +- app/models/milestone.rb | 2 +- app/models/project.rb | 28 +++---- app/models/todo.rb | 2 +- app/models/user.rb | 22 +++--- app/services/ci/register_job_service.rb | 2 +- rubocop/cop/gitlab/has_many_through_scope.rb | 45 ----------- rubocop/rubocop.rb | 3 +- .../cop/gitlab/has_many_through_scope_spec.rb | 74 ------------------- 21 files changed, 43 insertions(+), 169 deletions(-) delete mode 100644 rubocop/cop/gitlab/has_many_through_scope.rb delete mode 100644 spec/rubocop/cop/gitlab/has_many_through_scope_spec.rb diff --git a/Gemfile b/Gemfile index 1c822a40fc6..ab4d57d8b3c 100644 --- a/Gemfile +++ b/Gemfile @@ -434,5 +434,3 @@ gem 'grape_logging', '~> 1.7' # Asset synchronization gem 'asset_sync', '~> 2.2.0' - -gem 'goldiloader', '~> 2.0' diff --git a/Gemfile.lock b/Gemfile.lock index 7f243491c90..c2b629a1ecc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -331,9 +331,6 @@ GEM rubyntlm (~> 0.5) globalid (0.4.1) activesupport (>= 4.2.0) - goldiloader (2.0.1) - activerecord (>= 4.2, < 5.2) - activesupport (>= 4.2, < 5.2) gollum-grit_adapter (1.0.1) gitlab-grit (~> 2.7, >= 2.7.1) gon (6.1.0) @@ -1072,7 +1069,6 @@ DEPENDENCIES gitlab-markup (~> 1.6.2) gitlab-styles (~> 2.3) gitlab_omniauth-ldap (~> 2.0.4) - goldiloader (~> 2.0) gon (~> 6.1.0) google-api-client (~> 0.19.8) google-protobuf (= 3.5.1) diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index ee0d8df8eb7..5a4c56ec0dc 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -13,7 +13,7 @@ module Ci has_many :builds has_many :runner_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, -> { auto_include(false) }, through: :runner_projects + has_many :projects, through: :runner_projects has_one :last_build, ->() { order('id DESC') }, class_name: 'Ci::Build' diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index e4a06f3f976..77947d515c1 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -15,7 +15,7 @@ module Clusters belongs_to :user has_many :cluster_projects, class_name: 'Clusters::Project' - has_many :projects, -> { auto_include(false) }, through: :cluster_projects, class_name: '::Project' + has_many :projects, through: :cluster_projects, class_name: '::Project' # we force autosave to happen when we save `Cluster` model has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index d9416352f9c..b45395343cc 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -48,7 +48,7 @@ module Issuable end has_many :label_links, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :labels, -> { auto_include(false) }, through: :label_links + has_many :labels, through: :label_links has_many :todos, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :metrics diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 399abb67c9d..7c236369793 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -102,7 +102,7 @@ module ResolvableDiscussion yield(notes_relation) # Set the notes array to the updated notes - @notes = notes_relation.fresh.auto_include(false).to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables + @notes = notes_relation.fresh.to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables self.class.memoized_values.each do |name| clear_memoization(name) diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb index 858b7ef533e..89a74b7dcb1 100644 --- a/app/models/deploy_key.rb +++ b/app/models/deploy_key.rb @@ -2,7 +2,7 @@ class DeployKey < Key include IgnorableColumn has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, -> { auto_include(false) }, through: :deploy_keys_projects + has_many :projects, through: :deploy_keys_projects scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) } scope :are_public, -> { where(public: true) } diff --git a/app/models/deploy_token.rb b/app/models/deploy_token.rb index 8dae821a10e..979e9232fda 100644 --- a/app/models/deploy_token.rb +++ b/app/models/deploy_token.rb @@ -8,7 +8,7 @@ class DeployToken < ActiveRecord::Base default_value_for(:expires_at) { Forever.date } has_many :project_deploy_tokens, inverse_of: :deploy_token - has_many :projects, -> { auto_include(false) }, through: :project_deploy_tokens + has_many :projects, through: :project_deploy_tokens validate :ensure_at_least_one_scope before_save :ensure_token diff --git a/app/models/fork_network.rb b/app/models/fork_network.rb index aad3509b895..7f1728e8c77 100644 --- a/app/models/fork_network.rb +++ b/app/models/fork_network.rb @@ -1,7 +1,7 @@ class ForkNetwork < ActiveRecord::Base belongs_to :root_project, class_name: 'Project' has_many :fork_network_members - has_many :projects, -> { auto_include(false) }, through: :fork_network_members + has_many :projects, through: :fork_network_members after_create :add_root_as_member, if: :root_project diff --git a/app/models/group.rb b/app/models/group.rb index 202988d743d..8ff781059cc 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -12,9 +12,9 @@ class Group < Namespace has_many :group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members - has_many :users, -> { auto_include(false) }, through: :group_members + has_many :users, through: :group_members has_many :owners, - -> { where(members: { access_level: Gitlab::Access::OWNER }).auto_include(false) }, + -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :user @@ -23,7 +23,7 @@ class Group < Namespace has_many :milestones has_many :project_group_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :shared_projects, -> { auto_include(false) }, through: :project_group_links, source: :project + has_many :shared_projects, through: :project_group_links, source: :project has_many :notification_settings, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent has_many :labels, class_name: 'GroupLabel' has_many :variables, class_name: 'Ci::GroupVariable' diff --git a/app/models/issue.rb b/app/models/issue.rb index 702bfc77803..7611e83647c 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -34,7 +34,7 @@ class Issue < ActiveRecord::Base dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees - has_many :assignees, -> { auto_include(false) }, class_name: "User", through: :issue_assignees + has_many :assignees, class_name: "User", through: :issue_assignees validates :project, presence: true diff --git a/app/models/label.rb b/app/models/label.rb index f3496884cff..de7f1d56c64 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -18,8 +18,8 @@ class Label < ActiveRecord::Base has_many :lists, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :priorities, class_name: 'LabelPriority' has_many :label_links, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :issues, -> { auto_include(false) }, through: :label_links, source: :target, source_type: 'Issue' - has_many :merge_requests, -> { auto_include(false) }, through: :label_links, source: :target, source_type: 'MergeRequest' + has_many :issues, through: :label_links, source: :target, source_type: 'Issue' + has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' before_validation :strip_whitespace_from_title_and_color diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index ed95613ee59..b7de46fa202 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -3,7 +3,7 @@ class LfsObject < ActiveRecord::Base include ObjectStorage::BackgroundMove has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :projects, -> { auto_include(false) }, through: :lfs_objects_projects + has_many :projects, through: :lfs_objects_projects scope :with_files_stored_locally, -> { where(file_store: [nil, LfsObjectUploader::Store::LOCAL]) } diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 8e33bab81c1..a66a0015827 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -22,7 +22,7 @@ class Milestone < ActiveRecord::Base belongs_to :group has_many :issues - has_many :labels, -> { distinct.reorder('labels.title').auto_include(false) }, through: :issues + has_many :labels, -> { distinct.reorder('labels.title') }, through: :issues has_many :merge_requests has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent diff --git a/app/models/project.rb b/app/models/project.rb index 38139a9b137..5c37cd85243 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -138,11 +138,11 @@ class Project < ActiveRecord::Base has_one :packagist_service # TODO: replace these relations with the fork network versions - has_one :forked_project_link, foreign_key: "forked_to_project_id" - has_one :forked_from_project, -> { auto_include(false) }, through: :forked_project_link + has_one :forked_project_link, foreign_key: "forked_to_project_id" + has_one :forked_from_project, through: :forked_project_link has_many :forked_project_links, foreign_key: "forked_from_project_id" - has_many :forks, -> { auto_include(false) }, through: :forked_project_links, source: :forked_to_project + has_many :forks, through: :forked_project_links, source: :forked_to_project # TODO: replace these relations with the fork network versions has_one :root_of_fork_network, @@ -150,7 +150,7 @@ class Project < ActiveRecord::Base inverse_of: :root_project, class_name: 'ForkNetwork' has_one :fork_network_member - has_one :fork_network, -> { auto_include(false) }, through: :fork_network_member + has_one :fork_network, through: :fork_network_member # Merge Requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id' @@ -167,27 +167,27 @@ class Project < ActiveRecord::Base has_many :protected_tags has_many :project_authorizations - has_many :authorized_users, -> { auto_include(false) }, through: :project_authorizations, source: :user, class_name: 'User' + has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' has_many :project_members, -> { where(requested_at: nil) }, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :project_members - has_many :users, -> { auto_include(false) }, through: :project_members + has_many :users, through: :project_members has_many :requesters, -> { where.not(requested_at: nil) }, as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :members_and_requesters, as: :source, class_name: 'ProjectMember' has_many :deploy_keys_projects - has_many :deploy_keys, -> { auto_include(false) }, through: :deploy_keys_projects + has_many :deploy_keys, through: :deploy_keys_projects has_many :users_star_projects - has_many :starrers, -> { auto_include(false) }, through: :users_star_projects, source: :user + has_many :starrers, through: :users_star_projects, source: :user has_many :releases has_many :lfs_objects_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :lfs_objects, -> { auto_include(false) }, through: :lfs_objects_projects + has_many :lfs_objects, through: :lfs_objects_projects has_many :lfs_file_locks has_many :project_group_links - has_many :invited_groups, -> { auto_include(false) }, through: :project_group_links, source: :group + has_many :invited_groups, through: :project_group_links, source: :group has_many :pages_domains has_many :todos has_many :notification_settings, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -199,7 +199,7 @@ class Project < ActiveRecord::Base has_one :statistics, class_name: 'ProjectStatistics' has_one :cluster_project, class_name: 'Clusters::Project' - has_many :clusters, -> { auto_include(false) }, through: :cluster_project, class_name: 'Clusters::Cluster' + has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' # Container repositories need to remove data from the container registry, # which is not managed by the DB. Hence we're still using dependent: :destroy @@ -216,16 +216,16 @@ class Project < ActiveRecord::Base has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :build_trace_section_names, class_name: 'Ci::BuildTraceSectionName' has_many :runner_projects, class_name: 'Ci::RunnerProject' - has_many :runners, -> { auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_many :variables, class_name: 'Ci::Variable' has_many :triggers, class_name: 'Ci::Trigger' has_many :environments has_many :deployments has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule' has_many :project_deploy_tokens - has_many :deploy_tokens, -> { auto_include(false) }, through: :project_deploy_tokens + has_many :deploy_tokens, through: :project_deploy_tokens - has_many :active_runners, -> { active.auto_include(false) }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' + has_many :active_runners, -> { active }, through: :runner_projects, source: :runner, class_name: 'Ci::Runner' has_one :auto_devops, class_name: 'ProjectAutoDevops' has_many :custom_attributes, class_name: 'ProjectCustomAttribute' diff --git a/app/models/todo.rb b/app/models/todo.rb index aad2c1dac4e..a2ab405fdbe 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -22,7 +22,7 @@ class Todo < ActiveRecord::Base belongs_to :author, class_name: "User" belongs_to :note belongs_to :project - belongs_to :target, -> { auto_include(false) }, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations + belongs_to :target, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :user delegate :name, :email, to: :author, prefix: true, allow_nil: true diff --git a/app/models/user.rb b/app/models/user.rb index d5c5c0964c5..7a42d546b9c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -96,23 +96,23 @@ class User < ActiveRecord::Base # Groups has_many :members has_many :group_members, -> { where(requested_at: nil) }, source: 'GroupMember' - has_many :groups, -> { auto_include(false) }, through: :group_members - has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }).auto_include(false) }, through: :group_members, source: :group - has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }).auto_include(false) }, through: :group_members, source: :group + has_many :groups, through: :group_members + has_many :owned_groups, -> { where(members: { access_level: Gitlab::Access::OWNER }) }, through: :group_members, source: :group + has_many :masters_groups, -> { where(members: { access_level: Gitlab::Access::MASTER }) }, through: :group_members, source: :group # Projects - has_many :groups_projects, -> { auto_include(false) }, through: :groups, source: :projects - has_many :personal_projects, -> { auto_include(false) }, through: :namespace, source: :projects + has_many :groups_projects, through: :groups, source: :projects + has_many :personal_projects, through: :namespace, source: :projects has_many :project_members, -> { where(requested_at: nil) } - has_many :projects, -> { auto_include(false) }, through: :project_members - has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' + has_many :projects, through: :project_members + has_many :created_projects, foreign_key: :creator_id, class_name: 'Project' has_many :users_star_projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :starred_projects, -> { auto_include(false) }, through: :users_star_projects, source: :project + has_many :starred_projects, through: :users_star_projects, source: :project has_many :project_authorizations - has_many :authorized_projects, -> { auto_include(false) }, through: :project_authorizations, source: :project + has_many :authorized_projects, through: :project_authorizations, source: :project has_many :user_interacted_projects - has_many :project_interactions, -> { auto_include(false) }, through: :user_interacted_projects, source: :project, class_name: 'Project' + has_many :project_interactions, through: :user_interacted_projects, source: :project, class_name: 'Project' has_many :snippets, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :notes, dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent @@ -132,7 +132,7 @@ class User < ActiveRecord::Base has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id # rubocop:disable Cop/ActiveRecordDependent has_many :issue_assignees - has_many :assigned_issues, -> { auto_include(false) }, class_name: "Issue", through: :issue_assignees, source: :issue + has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue has_many :assigned_merge_requests, dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent has_many :custom_attributes, class_name: 'UserCustomAttribute' diff --git a/app/services/ci/register_job_service.rb b/app/services/ci/register_job_service.rb index c289b44f885..0b087ad73da 100644 --- a/app/services/ci/register_job_service.rb +++ b/app/services/ci/register_job_service.rb @@ -33,7 +33,7 @@ module Ci end end - builds.auto_include(false).find do |build| + builds.find do |build| next unless runner.can_pick?(build) begin diff --git a/rubocop/cop/gitlab/has_many_through_scope.rb b/rubocop/cop/gitlab/has_many_through_scope.rb deleted file mode 100644 index 770a2a0529f..00000000000 --- a/rubocop/cop/gitlab/has_many_through_scope.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'gitlab/styles/rubocop/model_helpers' - -module RuboCop - module Cop - module Gitlab - class HasManyThroughScope < RuboCop::Cop::Cop - include ::Gitlab::Styles::Rubocop::ModelHelpers - - MSG = 'Always provide an explicit scope calling auto_include(false) when using has_many :through'.freeze - - def_node_search :through?, <<~PATTERN - (pair (sym :through) _) - PATTERN - - def_node_matcher :has_many_through?, <<~PATTERN - (send nil? :has_many ... #through?) - PATTERN - - def_node_search :disables_auto_include?, <<~PATTERN - (send _ :auto_include false) - PATTERN - - def_node_matcher :scope_disables_auto_include?, <<~PATTERN - (block (send nil? :lambda) _ #disables_auto_include?) - PATTERN - - def on_send(node) - return unless in_model?(node) - return unless has_many_through?(node) - - target = node - scope_argument = node.children[3] - - if scope_argument.children[0].children.last == :lambda - return if scope_disables_auto_include?(scope_argument) - - target = scope_argument - end - - add_offense(target, location: :expression) - end - end - end - end -end diff --git a/rubocop/rubocop.rb b/rubocop/rubocop.rb index 96061672749..f05990232ab 100644 --- a/rubocop/rubocop.rb +++ b/rubocop/rubocop.rb @@ -1,8 +1,7 @@ # rubocop:disable Naming/FileName -require_relative 'cop/gitlab/has_many_through_scope' -require_relative 'cop/gitlab/httparty' require_relative 'cop/gitlab/module_with_instance_variables' require_relative 'cop/gitlab/predicate_memoization' +require_relative 'cop/gitlab/httparty' require_relative 'cop/include_sidekiq_worker' require_relative 'cop/avoid_return_from_blocks' require_relative 'cop/avoid_break_from_strong_memoize' diff --git a/spec/rubocop/cop/gitlab/has_many_through_scope_spec.rb b/spec/rubocop/cop/gitlab/has_many_through_scope_spec.rb deleted file mode 100644 index 6d769c8e6fd..00000000000 --- a/spec/rubocop/cop/gitlab/has_many_through_scope_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -require 'spec_helper' - -require 'rubocop' -require 'rubocop/rspec/support' - -require_relative '../../../../rubocop/cop/gitlab/has_many_through_scope' - -describe RuboCop::Cop::Gitlab::HasManyThroughScope do # rubocop:disable RSpec/FilePath - include CopHelper - - subject(:cop) { described_class.new } - - context 'in a model file' do - before do - allow(cop).to receive(:in_model?).and_return(true) - end - - context 'when the model does not use has_many :through' do - it 'does not register an offense' do - expect_no_offenses(<<-RUBY) - class User < ActiveRecord::Base - has_many :tags, source: 'UserTag' - end - RUBY - end - end - - context 'when the model uses has_many :through' do - context 'when the association has no scope defined' do - it 'registers an offense on the association' do - expect_offense(<<-RUBY) - class User < ActiveRecord::Base - has_many :tags, through: :user_tags - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #{described_class::MSG} - end - RUBY - end - end - - context 'when the association has a scope defined' do - context 'when the scope does not disable auto-loading' do - it 'registers an offense on the scope' do - expect_offense(<<-RUBY) - class User < ActiveRecord::Base - has_many :tags, -> { where(active: true) }, through: :user_tags - ^^^^^^^^^^^^^^^^^^^^^^^^^^ #{described_class::MSG} - end - RUBY - end - end - - context 'when the scope has auto_include(false)' do - it 'does not register an offense' do - expect_no_offenses(<<-RUBY) - class User < ActiveRecord::Base - has_many :tags, -> { where(active: true).auto_include(false).reorder(nil) }, through: :user_tags - end - RUBY - end - end - end - end - end - - context 'outside of a migration spec file' do - it 'does not register an offense' do - expect_no_offenses(<<-RUBY) - class User < ActiveRecord::Base - has_many :tags, through: :user_tags - end - RUBY - end - end -end