2018-07-25 05:30:33 -04:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2019-03-28 09:17:42 -04:00
|
|
|
class Snippet < ApplicationRecord
|
2014-10-08 09:44:25 -04:00
|
|
|
include Gitlab::VisibilityLevel
|
2018-09-20 10:14:46 -04:00
|
|
|
include Redactable
|
2016-10-06 17:17:11 -04:00
|
|
|
include CacheMarkdownField
|
2017-03-09 20:29:11 -05:00
|
|
|
include Noteable
|
2015-04-21 09:23:20 -04:00
|
|
|
include Participable
|
2015-05-02 23:11:21 -04:00
|
|
|
include Sortable
|
2016-06-03 05:44:04 -04:00
|
|
|
include Awardable
|
2016-11-17 15:46:31 -05:00
|
|
|
include Mentionable
|
2017-02-01 13:15:59 -05:00
|
|
|
include Spammable
|
2017-05-18 08:24:34 -04:00
|
|
|
include Editable
|
2017-11-24 05:45:19 -05:00
|
|
|
include Gitlab::SQL::Pattern
|
2018-09-11 11:31:34 -04:00
|
|
|
include FromUnion
|
2020-01-16 10:08:41 -05:00
|
|
|
include IgnorableColumns
|
2020-02-13 10:08:52 -05:00
|
|
|
include HasRepository
|
2020-12-14 13:09:48 -05:00
|
|
|
include CanMoveRepositoryStorage
|
2020-04-22 08:09:29 -04:00
|
|
|
include AfterCommitQueue
|
2019-09-24 23:06:21 -04:00
|
|
|
extend ::Gitlab::Utils::Override
|
2011-10-20 15:00:00 -04:00
|
|
|
|
2020-05-19 08:08:21 -04:00
|
|
|
MAX_FILE_COUNT = 10
|
2020-03-17 02:09:21 -04:00
|
|
|
|
2016-10-06 17:17:11 -04:00
|
|
|
cache_markdown_field :title, pipeline: :single_line
|
2017-05-03 11:26:49 -04:00
|
|
|
cache_markdown_field :description
|
2016-10-06 17:17:11 -04:00
|
|
|
cache_markdown_field :content
|
|
|
|
|
2018-09-20 10:14:46 -04:00
|
|
|
redact_field :description
|
|
|
|
|
2017-05-03 23:54:25 -04:00
|
|
|
# Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with snippets.
|
2019-09-18 10:02:45 -04:00
|
|
|
# See https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/10392/diffs#note_28719102
|
2017-05-03 23:54:25 -04:00
|
|
|
alias_attribute :last_edited_at, :updated_at
|
|
|
|
alias_attribute :last_edited_by, :updated_by
|
|
|
|
|
2016-10-06 17:17:11 -04:00
|
|
|
# If file_name changes, it invalidates content
|
|
|
|
alias_method :default_content_html_invalidator, :content_html_invalidated?
|
|
|
|
def content_html_invalidated?
|
|
|
|
default_content_html_invalidator || file_name_changed?
|
|
|
|
end
|
|
|
|
|
2015-05-02 23:14:31 -04:00
|
|
|
belongs_to :author, class_name: 'User'
|
|
|
|
belongs_to :project
|
2013-03-25 07:58:09 -04:00
|
|
|
|
2017-06-08 11:16:27 -04:00
|
|
|
has_many :notes, as: :noteable, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
|
2020-02-13 19:09:07 -05:00
|
|
|
has_many :user_mentions, class_name: "SnippetUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
|
2020-02-13 10:08:52 -05:00
|
|
|
has_one :snippet_repository, inverse_of: :snippet
|
2021-02-23 07:10:56 -05:00
|
|
|
has_many :repository_storage_moves, class_name: 'Snippets::RepositoryStorageMove', inverse_of: :container
|
2020-07-08 05:09:17 -04:00
|
|
|
|
|
|
|
# We need to add the `dependent` in order to call the after_destroy callback
|
|
|
|
has_one :statistics, class_name: 'SnippetStatistics', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
|
2011-10-16 17:07:10 -04:00
|
|
|
|
2012-12-14 00:34:05 -05:00
|
|
|
delegate :name, :email, to: :author, prefix: true, allow_nil: true
|
2011-10-16 17:07:10 -04:00
|
|
|
|
2012-10-08 20:10:04 -04:00
|
|
|
validates :author, presence: true
|
2016-12-02 07:54:57 -05:00
|
|
|
validates :title, presence: true, length: { maximum: 255 }
|
2015-02-03 00:15:44 -05:00
|
|
|
validates :file_name,
|
2017-07-05 12:01:38 -04:00
|
|
|
length: { maximum: 255 }
|
2016-06-16 14:55:04 -04:00
|
|
|
|
2013-01-22 10:10:00 -05:00
|
|
|
validates :content, presence: true
|
2019-12-02 07:06:45 -05:00
|
|
|
validates :content,
|
|
|
|
length: {
|
|
|
|
maximum: ->(_) { Gitlab::CurrentSettings.snippet_size_limit },
|
|
|
|
message: -> (_, data) do
|
|
|
|
current_value = ActiveSupport::NumberHelper.number_to_human_size(data[:value].size)
|
|
|
|
max_size = ActiveSupport::NumberHelper.number_to_human_size(Gitlab::CurrentSettings.snippet_size_limit)
|
|
|
|
|
|
|
|
_("is too long (%{current_value}). The maximum size is %{max_size}.") % { current_value: current_value, max_size: max_size }
|
|
|
|
end
|
|
|
|
},
|
|
|
|
if: :content_changed?
|
|
|
|
|
2014-10-08 09:44:25 -04:00
|
|
|
validates :visibility_level, inclusion: { in: Gitlab::VisibilityLevel.values }
|
2011-10-16 17:07:10 -04:00
|
|
|
|
2020-06-23 11:08:41 -04:00
|
|
|
after_create :create_statistics
|
2020-02-13 19:09:07 -05:00
|
|
|
|
2012-10-08 20:10:04 -04:00
|
|
|
# Scopes
|
2019-01-16 07:09:29 -05:00
|
|
|
scope :are_internal, -> { where(visibility_level: Snippet::INTERNAL) }
|
2014-10-08 09:44:25 -04:00
|
|
|
scope :are_private, -> { where(visibility_level: Snippet::PRIVATE) }
|
2019-11-19 01:06:07 -05:00
|
|
|
scope :are_public, -> { public_only }
|
|
|
|
scope :are_secret, -> { public_only.where(secret: true) }
|
2019-01-16 07:09:29 -05:00
|
|
|
scope :fresh, -> { order("created_at DESC") }
|
2019-09-02 07:12:20 -04:00
|
|
|
scope :inc_author, -> { includes(:author) }
|
2018-07-16 12:18:52 -04:00
|
|
|
scope :inc_relations_for_view, -> { includes(author: :status) }
|
2021-03-15 20:09:44 -04:00
|
|
|
scope :inc_statistics, -> { includes(:statistics) }
|
2020-06-26 11:08:45 -04:00
|
|
|
scope :with_statistics, -> { joins(:statistics) }
|
2021-01-12 13:11:03 -05:00
|
|
|
scope :inc_projects_namespace_route, -> { includes(project: [:route, :namespace]) }
|
2011-10-27 03:14:50 -04:00
|
|
|
|
2019-12-10 10:07:52 -05:00
|
|
|
attr_mentionable :description
|
|
|
|
|
2016-05-26 07:38:28 -04:00
|
|
|
participant :author
|
|
|
|
participant :notes_with_associations
|
2015-04-21 09:23:20 -04:00
|
|
|
|
2017-02-01 13:15:59 -05:00
|
|
|
attr_spammable :title, spam_title: true
|
|
|
|
attr_spammable :content, spam_description: true
|
|
|
|
|
2019-11-19 01:06:07 -05:00
|
|
|
attr_encrypted :secret_token,
|
|
|
|
key: Settings.attr_encrypted_db_key_base_truncated,
|
|
|
|
mode: :per_attribute_iv,
|
|
|
|
algorithm: 'aes-256-cbc'
|
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
class << self
|
|
|
|
# Searches for snippets with a matching title, description or file name.
|
|
|
|
#
|
|
|
|
# This method uses ILIKE on PostgreSQL.
|
|
|
|
#
|
|
|
|
# query - The search query as a String.
|
|
|
|
#
|
|
|
|
# Returns an ActiveRecord::Relation.
|
|
|
|
def search(query)
|
|
|
|
fuzzy_search(query, [:title, :description, :file_name])
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
end
|
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def parent_class
|
|
|
|
::Project
|
|
|
|
end
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def sanitized_file_name(file_name)
|
|
|
|
file_name.gsub(/[^a-zA-Z0-9_\-\.]+/, '')
|
|
|
|
end
|
2020-04-27 08:09:41 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def with_optional_visibility(value = nil)
|
|
|
|
if value
|
|
|
|
where(visibility_level: value)
|
|
|
|
else
|
|
|
|
all
|
|
|
|
end
|
|
|
|
end
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def only_personal_snippets
|
|
|
|
where(project_id: nil)
|
|
|
|
end
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def only_project_snippets
|
|
|
|
where.not(project_id: nil)
|
|
|
|
end
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def only_include_projects_visible_to(current_user = nil)
|
|
|
|
levels = Gitlab::VisibilityLevel.levels_for_user(current_user)
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
joins(:project).where(projects: { visibility_level: levels })
|
|
|
|
end
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def only_include_projects_with_snippets_enabled(include_private: false)
|
|
|
|
column = ProjectFeature.access_level_attribute(:snippets)
|
|
|
|
levels = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC]
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
levels << ProjectFeature::PRIVATE if include_private
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
joins(project: :project_feature)
|
|
|
|
.where(project_features: { column => levels })
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
end
|
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def only_include_authorized_projects(current_user)
|
|
|
|
where(
|
|
|
|
'EXISTS (?)',
|
|
|
|
ProjectAuthorization
|
|
|
|
.select(1)
|
|
|
|
.where('project_id = snippets.project_id')
|
|
|
|
.where(user_id: current_user.id)
|
|
|
|
)
|
|
|
|
end
|
Rewrite SnippetsFinder to improve performance
This completely rewrites the SnippetsFinder class from the ground up in
order to improve its performance. The old code was beyond salvaging. It
was complex, included various Rails 5 workarounds, comments that
shouldn't be necessary, and most important of all: it produced a really
poorly performing database query.
As a result, I opted for rewriting the finder from scratch, instead of
trying to patch the existing code. Instead of trying to reuse as many
existing methods as possible, I opted for defining new methods
specifically meant for the SnippetsFinder. This requires some extra code
here and there, but allows us to have much more control over the
resulting SQL queries. It is these changes that then allow us to produce
a _much_ more efficient query.
To illustrate how bad the old query was, we will use my own snippets as
an example. Currently I have 52 snippets, most of which are global ones.
To retrieve these, you would run the following Ruby code:
user = User.find_by(username: 'yorickpeterse')
SnippetsFinder.new(user, author: user).execute
On GitLab.com the resulting query will take between 10 and 15 seconds to
run, producing the query plan found at
https://explain.depesz.com/s/Y5IX. Apart from the long execution time,
the total number of buffers (the sum of all shared hits) is around 185
GB, though the real number is probably (hopefully) much lower as I doubt
simply summing these numbers produces the true total number of buffers
used.
The new query's plan can be found at https://explain.depesz.com/s/wHdN,
and this query takes between 10 and 100-ish milliseconds to run. The
total number of buffers used is only about 30 MB.
Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def for_project_with_user(project, user = nil)
|
|
|
|
return none unless project.snippets_visible?(user)
|
|
|
|
|
|
|
|
if user && project.team.member?(user)
|
|
|
|
project.snippets
|
|
|
|
else
|
|
|
|
project.snippets.public_to_user(user)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
def visible_to_or_authored_by(user)
|
|
|
|
query = where(visibility_level: Gitlab::VisibilityLevel.levels_for_user(user))
|
|
|
|
query.or(where(author_id: user.id))
|
|
|
|
end
|
|
|
|
|
|
|
|
def reference_prefix
|
|
|
|
'$'
|
|
|
|
end
|
2015-05-02 23:11:21 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
# Pattern used to extract `$123` snippet references from text
|
|
|
|
#
|
|
|
|
# This pattern supports cross-project references.
|
|
|
|
def reference_pattern
|
|
|
|
@reference_pattern ||= %r{
|
2015-12-01 09:51:27 -05:00
|
|
|
(#{Project.reference_pattern})?
|
|
|
|
#{Regexp.escape(reference_prefix)}(?<snippet>\d+)
|
2015-05-14 16:59:39 -04:00
|
|
|
}x
|
2021-11-30 10:14:19 -05:00
|
|
|
end
|
2015-05-14 16:59:39 -04:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def link_reference_pattern
|
|
|
|
@link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/)
|
|
|
|
end
|
2015-11-30 15:14:46 -05:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def find_by_id_and_project(id:, project:)
|
|
|
|
Snippet.find_by(id: id, project: project)
|
|
|
|
end
|
2020-03-04 19:07:49 -05:00
|
|
|
|
2021-11-30 10:14:19 -05:00
|
|
|
def max_file_limit
|
|
|
|
MAX_FILE_COUNT
|
|
|
|
end
|
2020-05-19 08:08:21 -04:00
|
|
|
end
|
|
|
|
|
2019-10-16 20:07:27 -04:00
|
|
|
def initialize(attributes = {})
|
|
|
|
# We can't use default_value_for because the database has a default
|
|
|
|
# value of 0 for visibility_level. If someone attempts to create a
|
|
|
|
# private snippet, default_value_for will assume that the
|
|
|
|
# visibility_level hasn't changed and will use the application
|
|
|
|
# setting default, which could be internal or public.
|
|
|
|
#
|
|
|
|
# To fix the problem, we assign the actual snippet default if no
|
|
|
|
# explicit visibility has been initialized.
|
|
|
|
attributes ||= {}
|
|
|
|
|
|
|
|
unless visibility_attribute_present?(attributes)
|
|
|
|
attributes[:visibility_level] = Gitlab::CurrentSettings.default_snippet_visibility
|
|
|
|
end
|
|
|
|
|
|
|
|
super
|
|
|
|
end
|
|
|
|
|
2017-11-22 08:20:35 -05:00
|
|
|
def to_reference(from = nil, full: false)
|
2015-05-02 23:11:21 -04:00
|
|
|
reference = "#{self.class.reference_prefix}#{id}"
|
|
|
|
|
2016-11-02 19:49:13 -04:00
|
|
|
if project.present?
|
2020-01-29 07:09:08 -05:00
|
|
|
"#{project.to_reference_base(from, full: full)}#{reference}"
|
2016-11-02 19:49:13 -04:00
|
|
|
else
|
|
|
|
reference
|
2015-05-02 23:11:21 -04:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2017-04-13 12:47:28 -04:00
|
|
|
def blob
|
2020-02-25 04:09:10 -05:00
|
|
|
@blob ||= Blob.decorate(SnippetBlob.new(self), self)
|
|
|
|
end
|
|
|
|
|
|
|
|
def blobs
|
2020-03-06 10:08:05 -05:00
|
|
|
return [] unless repository_exists?
|
|
|
|
|
2021-02-22 04:10:46 -05:00
|
|
|
files = list_files(default_branch)
|
|
|
|
items = files.map { |file| [default_branch, file] }
|
|
|
|
|
|
|
|
repository.blobs_at(items).compact
|
2012-04-20 18:26:22 -04:00
|
|
|
end
|
|
|
|
|
2015-03-05 13:38:23 -05:00
|
|
|
def hook_attrs
|
|
|
|
attributes
|
|
|
|
end
|
|
|
|
|
2016-12-02 07:54:57 -05:00
|
|
|
def file_name
|
|
|
|
super.to_s
|
|
|
|
end
|
|
|
|
|
2014-10-08 09:44:25 -04:00
|
|
|
def visibility_level_field
|
2017-03-01 15:23:00 -05:00
|
|
|
:visibility_level
|
2014-12-12 06:15:42 -05:00
|
|
|
end
|
2014-10-08 09:44:25 -04:00
|
|
|
|
2018-12-11 01:28:06 -05:00
|
|
|
def embeddable?
|
2020-01-23 07:08:38 -05:00
|
|
|
Ability.allowed?(nil, :read_snippet, self)
|
2018-12-11 01:28:06 -05:00
|
|
|
end
|
|
|
|
|
2016-05-26 07:38:28 -04:00
|
|
|
def notes_with_associations
|
2016-06-03 14:47:09 -04:00
|
|
|
notes.includes(:author)
|
2016-05-26 07:38:28 -04:00
|
|
|
end
|
|
|
|
|
2021-07-20 02:08:37 -04:00
|
|
|
def check_for_spam?(user:)
|
2017-03-20 22:37:29 -04:00
|
|
|
visibility_level_changed?(to: Snippet::PUBLIC) ||
|
|
|
|
(public? && (title_changed? || content_changed?))
|
2017-02-01 13:15:59 -05:00
|
|
|
end
|
|
|
|
|
2020-01-19 19:09:03 -05:00
|
|
|
# snippets are the biggest sources of spam
|
2019-09-24 23:06:21 -04:00
|
|
|
override :allow_possible_spam?
|
|
|
|
def allow_possible_spam?
|
|
|
|
false
|
|
|
|
end
|
|
|
|
|
2017-02-01 13:15:59 -05:00
|
|
|
def spammable_entity_type
|
|
|
|
'snippet'
|
|
|
|
end
|
|
|
|
|
2019-06-21 00:45:27 -04:00
|
|
|
def to_ability_name
|
2020-01-23 07:08:38 -05:00
|
|
|
'snippet'
|
2019-06-21 00:45:27 -04:00
|
|
|
end
|
|
|
|
|
2019-11-19 01:06:07 -05:00
|
|
|
def valid_secret_token?(token)
|
|
|
|
return false unless token && secret_token
|
|
|
|
|
|
|
|
ActiveSupport::SecurityUtils.secure_compare(token.to_s, secret_token.to_s)
|
|
|
|
end
|
|
|
|
|
|
|
|
def as_json(options = {})
|
|
|
|
options[:except] = Array.wrap(options[:except])
|
|
|
|
options[:except] << :secret_token
|
|
|
|
|
|
|
|
super
|
|
|
|
end
|
|
|
|
|
2020-04-07 11:09:30 -04:00
|
|
|
override :repository
|
2020-02-13 10:08:52 -05:00
|
|
|
def repository
|
2020-09-08 20:08:42 -04:00
|
|
|
@repository ||= Gitlab::GlRepository::SNIPPET.repository_for(self)
|
2020-02-13 10:08:52 -05:00
|
|
|
end
|
|
|
|
|
2020-04-07 11:09:30 -04:00
|
|
|
override :repository_size_checker
|
2020-04-06 20:09:33 -04:00
|
|
|
def repository_size_checker
|
|
|
|
strong_memoize(:repository_size_checker) do
|
|
|
|
::Gitlab::RepositorySizeChecker.new(
|
2020-04-09 11:09:29 -04:00
|
|
|
current_size_proc: -> { repository.size.megabytes },
|
2020-10-12 14:08:31 -04:00
|
|
|
limit: Gitlab::CurrentSettings.snippet_size_limit,
|
2020-10-19 08:09:20 -04:00
|
|
|
namespace: nil
|
2020-04-06 20:09:33 -04:00
|
|
|
)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-04-07 11:09:30 -04:00
|
|
|
override :storage
|
2020-02-13 10:08:52 -05:00
|
|
|
def storage
|
|
|
|
@storage ||= Storage::Hashed.new(self, prefix: Storage::Hashed::SNIPPET_REPOSITORY_PATH_PREFIX)
|
|
|
|
end
|
|
|
|
|
2020-10-27 14:08:59 -04:00
|
|
|
# This is the full_path used to identify the the snippet repository.
|
2020-04-07 11:09:30 -04:00
|
|
|
override :full_path
|
2020-02-13 10:08:52 -05:00
|
|
|
def full_path
|
|
|
|
return unless persisted?
|
|
|
|
|
|
|
|
@full_path ||= begin
|
|
|
|
components = []
|
|
|
|
components << project.full_path if project_id?
|
2020-10-27 14:08:59 -04:00
|
|
|
components << 'snippets'
|
2020-02-13 10:08:52 -05:00
|
|
|
components << self.id
|
|
|
|
components.join('/')
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2020-09-16 08:10:15 -04:00
|
|
|
override :default_branch
|
|
|
|
def default_branch
|
2021-05-14 11:10:35 -04:00
|
|
|
super || Gitlab::DefaultBranch.value(object: project)
|
2020-09-16 08:10:15 -04:00
|
|
|
end
|
|
|
|
|
2020-02-13 10:08:52 -05:00
|
|
|
def repository_storage
|
2021-01-25 10:09:00 -05:00
|
|
|
snippet_repository&.shard_name || Repository.pick_storage_shard
|
2020-02-13 10:08:52 -05:00
|
|
|
end
|
|
|
|
|
2021-05-13 08:10:02 -04:00
|
|
|
# Repositories are created with a default branch. This branch
|
|
|
|
# can be different from the default branch set in the platform.
|
2020-12-23 04:10:13 -05:00
|
|
|
# This method changes the `HEAD` file to point to the existing
|
2021-05-13 08:10:02 -04:00
|
|
|
# default branch in case it's different.
|
2020-12-23 04:10:13 -05:00
|
|
|
def change_head_to_default_branch
|
|
|
|
return unless repository.exists?
|
|
|
|
# All snippets must have at least 1 file. Therefore, if
|
|
|
|
# `HEAD` is empty is because it's pointing to the wrong
|
|
|
|
# default branch
|
|
|
|
return unless repository.empty? || list_files('HEAD').empty?
|
|
|
|
|
|
|
|
repository.raw_repository.write_ref('HEAD', "refs/heads/#{default_branch}")
|
|
|
|
end
|
|
|
|
|
2020-02-13 10:08:52 -05:00
|
|
|
def create_repository
|
2020-03-04 10:08:09 -05:00
|
|
|
return if repository_exists? && snippet_repository
|
2020-02-13 10:08:52 -05:00
|
|
|
|
|
|
|
repository.create_if_not_exists
|
2020-03-25 08:08:19 -04:00
|
|
|
track_snippet_repository(repository.storage)
|
2020-02-13 10:08:52 -05:00
|
|
|
end
|
|
|
|
|
2020-03-25 08:08:19 -04:00
|
|
|
def track_snippet_repository(shard)
|
|
|
|
snippet_repo = snippet_repository || build_snippet_repository
|
|
|
|
snippet_repo.update!(shard_name: shard, disk_path: disk_path)
|
2020-02-13 10:08:52 -05:00
|
|
|
end
|
|
|
|
|
2020-02-28 16:09:15 -05:00
|
|
|
def can_cache_field?(field)
|
|
|
|
field != :content || MarkupHelper.gitlab_markdown?(file_name)
|
|
|
|
end
|
|
|
|
|
2020-03-06 13:08:08 -05:00
|
|
|
def hexdigest
|
|
|
|
Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
|
|
|
|
end
|
|
|
|
|
2020-04-21 11:21:10 -04:00
|
|
|
def file_name_on_repo
|
|
|
|
return if repository.empty?
|
|
|
|
|
2020-09-16 08:10:15 -04:00
|
|
|
list_files(default_branch).first
|
2020-07-01 11:08:45 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
def list_files(ref = nil)
|
|
|
|
return [] if repository.empty?
|
|
|
|
|
2020-09-16 08:10:15 -04:00
|
|
|
repository.ls_files(ref || default_branch)
|
2020-04-21 11:21:10 -04:00
|
|
|
end
|
|
|
|
|
2020-09-02 05:10:23 -04:00
|
|
|
def multiple_files?
|
2020-09-16 08:10:15 -04:00
|
|
|
list_files.size > 1
|
2020-09-02 05:10:23 -04:00
|
|
|
end
|
2011-10-16 17:07:10 -04:00
|
|
|
end
|
2019-09-13 09:26:31 -04:00
|
|
|
|
2021-05-11 17:10:21 -04:00
|
|
|
Snippet.prepend_mod_with('Snippet')
|