gitlab-org--gitlab-foss/app/models/snippet.rb

371 lines
10 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
class Snippet < ApplicationRecord
2014-10-08 09:44:25 -04:00
include Gitlab::VisibilityLevel
include Redactable
include CacheMarkdownField
include Noteable
include Participable
include Sortable
2016-06-03 05:44:04 -04:00
include Awardable
include Mentionable
include Spammable
include Editable
include Gitlab::SQL::Pattern
include FromUnion
include IgnorableColumns
include HasRepository
include AfterCommitQueue
extend ::Gitlab::Utils::Override
2011-10-20 15:00:00 -04:00
MAX_FILE_COUNT = 10
MAX_SINGLE_FILE_COUNT = 1
cache_markdown_field :title, pipeline: :single_line
2017-05-03 11:26:49 -04:00
cache_markdown_field :description
cache_markdown_field :content
redact_field :description
2017-05-03 23:54:25 -04:00
# Aliases to make application_helper#edited_time_ago_with_tooltip helper work properly with snippets.
# See https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/10392/diffs#note_28719102
2017-05-03 23:54:25 -04:00
alias_attribute :last_edited_at, :updated_at
alias_attribute :last_edited_by, :updated_by
# If file_name changes, it invalidates content
alias_method :default_content_html_invalidator, :content_html_invalidated?
def content_html_invalidated?
default_content_html_invalidator || file_name_changed?
end
belongs_to :author, class_name: 'User'
belongs_to :project
2013-03-25 07:58:09 -04:00
has_many :notes, as: :noteable, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :user_mentions, class_name: "SnippetUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
has_one :snippet_repository, inverse_of: :snippet
# We need to add the `dependent` in order to call the after_destroy callback
has_one :statistics, class_name: 'SnippetStatistics', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
2011-10-16 17:07:10 -04:00
2012-12-14 00:34:05 -05:00
delegate :name, :email, to: :author, prefix: true, allow_nil: true
2011-10-16 17:07:10 -04:00
2012-10-08 20:10:04 -04:00
validates :author, presence: true
validates :title, presence: true, length: { maximum: 255 }
2015-02-03 00:15:44 -05:00
validates :file_name,
length: { maximum: 255 }
2016-06-16 14:55:04 -04:00
2013-01-22 10:10:00 -05:00
validates :content, presence: true
validates :content,
length: {
maximum: ->(_) { Gitlab::CurrentSettings.snippet_size_limit },
message: -> (_, data) do
current_value = ActiveSupport::NumberHelper.number_to_human_size(data[:value].size)
max_size = ActiveSupport::NumberHelper.number_to_human_size(Gitlab::CurrentSettings.snippet_size_limit)
_("is too long (%{current_value}). The maximum size is %{max_size}.") % { current_value: current_value, max_size: max_size }
end
},
if: :content_changed?
2014-10-08 09:44:25 -04:00
validates :visibility_level, inclusion: { in: Gitlab::VisibilityLevel.values }
2011-10-16 17:07:10 -04:00
after_save :store_mentions!, if: :any_mentionable_attributes_changed?
after_create :create_statistics
2012-10-08 20:10:04 -04:00
# Scopes
scope :are_internal, -> { where(visibility_level: Snippet::INTERNAL) }
2014-10-08 09:44:25 -04:00
scope :are_private, -> { where(visibility_level: Snippet::PRIVATE) }
scope :are_public, -> { public_only }
scope :are_secret, -> { public_only.where(secret: true) }
scope :fresh, -> { order("created_at DESC") }
scope :inc_author, -> { includes(:author) }
scope :inc_relations_for_view, -> { includes(author: :status) }
scope :with_statistics, -> { joins(:statistics) }
2011-10-27 03:14:50 -04:00
attr_mentionable :description
participant :author
participant :notes_with_associations
attr_spammable :title, spam_title: true
attr_spammable :content, spam_description: true
attr_encrypted :secret_token,
key: Settings.attr_encrypted_db_key_base_truncated,
mode: :per_attribute_iv,
algorithm: 'aes-256-cbc'
Rewrite SnippetsFinder to improve performance This completely rewrites the SnippetsFinder class from the ground up in order to improve its performance. The old code was beyond salvaging. It was complex, included various Rails 5 workarounds, comments that shouldn't be necessary, and most important of all: it produced a really poorly performing database query. As a result, I opted for rewriting the finder from scratch, instead of trying to patch the existing code. Instead of trying to reuse as many existing methods as possible, I opted for defining new methods specifically meant for the SnippetsFinder. This requires some extra code here and there, but allows us to have much more control over the resulting SQL queries. It is these changes that then allow us to produce a _much_ more efficient query. To illustrate how bad the old query was, we will use my own snippets as an example. Currently I have 52 snippets, most of which are global ones. To retrieve these, you would run the following Ruby code: user = User.find_by(username: 'yorickpeterse') SnippetsFinder.new(user, author: user).execute On GitLab.com the resulting query will take between 10 and 15 seconds to run, producing the query plan found at https://explain.depesz.com/s/Y5IX. Apart from the long execution time, the total number of buffers (the sum of all shared hits) is around 185 GB, though the real number is probably (hopefully) much lower as I doubt simply summing these numbers produces the true total number of buffers used. The new query's plan can be found at https://explain.depesz.com/s/wHdN, and this query takes between 10 and 100-ish milliseconds to run. The total number of buffers used is only about 30 MB. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
def self.with_optional_visibility(value = nil)
if value
where(visibility_level: value)
else
all
end
end
def self.only_personal_snippets
Rewrite SnippetsFinder to improve performance This completely rewrites the SnippetsFinder class from the ground up in order to improve its performance. The old code was beyond salvaging. It was complex, included various Rails 5 workarounds, comments that shouldn't be necessary, and most important of all: it produced a really poorly performing database query. As a result, I opted for rewriting the finder from scratch, instead of trying to patch the existing code. Instead of trying to reuse as many existing methods as possible, I opted for defining new methods specifically meant for the SnippetsFinder. This requires some extra code here and there, but allows us to have much more control over the resulting SQL queries. It is these changes that then allow us to produce a _much_ more efficient query. To illustrate how bad the old query was, we will use my own snippets as an example. Currently I have 52 snippets, most of which are global ones. To retrieve these, you would run the following Ruby code: user = User.find_by(username: 'yorickpeterse') SnippetsFinder.new(user, author: user).execute On GitLab.com the resulting query will take between 10 and 15 seconds to run, producing the query plan found at https://explain.depesz.com/s/Y5IX. Apart from the long execution time, the total number of buffers (the sum of all shared hits) is around 185 GB, though the real number is probably (hopefully) much lower as I doubt simply summing these numbers produces the true total number of buffers used. The new query's plan can be found at https://explain.depesz.com/s/wHdN, and this query takes between 10 and 100-ish milliseconds to run. The total number of buffers used is only about 30 MB. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
where(project_id: nil)
end
def self.only_project_snippets
where.not(project_id: nil)
end
Rewrite SnippetsFinder to improve performance This completely rewrites the SnippetsFinder class from the ground up in order to improve its performance. The old code was beyond salvaging. It was complex, included various Rails 5 workarounds, comments that shouldn't be necessary, and most important of all: it produced a really poorly performing database query. As a result, I opted for rewriting the finder from scratch, instead of trying to patch the existing code. Instead of trying to reuse as many existing methods as possible, I opted for defining new methods specifically meant for the SnippetsFinder. This requires some extra code here and there, but allows us to have much more control over the resulting SQL queries. It is these changes that then allow us to produce a _much_ more efficient query. To illustrate how bad the old query was, we will use my own snippets as an example. Currently I have 52 snippets, most of which are global ones. To retrieve these, you would run the following Ruby code: user = User.find_by(username: 'yorickpeterse') SnippetsFinder.new(user, author: user).execute On GitLab.com the resulting query will take between 10 and 15 seconds to run, producing the query plan found at https://explain.depesz.com/s/Y5IX. Apart from the long execution time, the total number of buffers (the sum of all shared hits) is around 185 GB, though the real number is probably (hopefully) much lower as I doubt simply summing these numbers produces the true total number of buffers used. The new query's plan can be found at https://explain.depesz.com/s/wHdN, and this query takes between 10 and 100-ish milliseconds to run. The total number of buffers used is only about 30 MB. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
def self.only_include_projects_visible_to(current_user = nil)
levels = Gitlab::VisibilityLevel.levels_for_user(current_user)
joins(:project).where('projects.visibility_level IN (?)', levels)
end
def self.only_include_projects_with_snippets_enabled(include_private: false)
column = ProjectFeature.access_level_attribute(:snippets)
levels = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC]
levels << ProjectFeature::PRIVATE if include_private
joins(project: :project_feature)
.where(project_features: { column => levels })
end
def self.only_include_authorized_projects(current_user)
where(
'EXISTS (?)',
ProjectAuthorization
.select(1)
.where('project_id = snippets.project_id')
.where(user_id: current_user.id)
)
end
def self.for_project_with_user(project, user = nil)
return none unless project.snippets_visible?(user)
if user && project.team.member?(user)
project.snippets
else
project.snippets.public_to_user(user)
end
end
def self.visible_to_or_authored_by(user)
query = where(visibility_level: Gitlab::VisibilityLevel.levels_for_user(user))
query.or(where(author_id: user.id))
Rewrite SnippetsFinder to improve performance This completely rewrites the SnippetsFinder class from the ground up in order to improve its performance. The old code was beyond salvaging. It was complex, included various Rails 5 workarounds, comments that shouldn't be necessary, and most important of all: it produced a really poorly performing database query. As a result, I opted for rewriting the finder from scratch, instead of trying to patch the existing code. Instead of trying to reuse as many existing methods as possible, I opted for defining new methods specifically meant for the SnippetsFinder. This requires some extra code here and there, but allows us to have much more control over the resulting SQL queries. It is these changes that then allow us to produce a _much_ more efficient query. To illustrate how bad the old query was, we will use my own snippets as an example. Currently I have 52 snippets, most of which are global ones. To retrieve these, you would run the following Ruby code: user = User.find_by(username: 'yorickpeterse') SnippetsFinder.new(user, author: user).execute On GitLab.com the resulting query will take between 10 and 15 seconds to run, producing the query plan found at https://explain.depesz.com/s/Y5IX. Apart from the long execution time, the total number of buffers (the sum of all shared hits) is around 185 GB, though the real number is probably (hopefully) much lower as I doubt simply summing these numbers produces the true total number of buffers used. The new query's plan can be found at https://explain.depesz.com/s/wHdN, and this query takes between 10 and 100-ish milliseconds to run. The total number of buffers used is only about 30 MB. Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/52639
2018-10-25 11:35:31 -04:00
end
def self.reference_prefix
'$'
end
# Pattern used to extract `$123` snippet references from text
#
# This pattern supports cross-project references.
def self.reference_pattern
@reference_pattern ||= %r{
(#{Project.reference_pattern})?
#{Regexp.escape(reference_prefix)}(?<snippet>\d+)
}x
end
def self.link_reference_pattern
@link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/)
end
def self.find_by_id_and_project(id:, project:)
Snippet.find_by(id: id, project: project)
end
def self.max_file_limit(user)
Feature.enabled?(:snippet_multiple_files, user) ? MAX_FILE_COUNT : MAX_SINGLE_FILE_COUNT
end
def initialize(attributes = {})
# We can't use default_value_for because the database has a default
# value of 0 for visibility_level. If someone attempts to create a
# private snippet, default_value_for will assume that the
# visibility_level hasn't changed and will use the application
# setting default, which could be internal or public.
#
# To fix the problem, we assign the actual snippet default if no
# explicit visibility has been initialized.
attributes ||= {}
unless visibility_attribute_present?(attributes)
attributes[:visibility_level] = Gitlab::CurrentSettings.default_snippet_visibility
end
super
end
2017-11-22 08:20:35 -05:00
def to_reference(from = nil, full: false)
reference = "#{self.class.reference_prefix}#{id}"
if project.present?
"#{project.to_reference_base(from, full: full)}#{reference}"
else
reference
end
end
2017-04-13 12:47:28 -04:00
def blob
@blob ||= Blob.decorate(SnippetBlob.new(self), self)
end
def blobs
return [] unless repository_exists?
repository.ls_files(repository.root_ref).map { |file| Blob.lazy(repository, repository.root_ref, file) }
2012-04-20 18:26:22 -04:00
end
def hook_attrs
attributes
end
def file_name
super.to_s
end
def self.sanitized_file_name(file_name)
file_name.gsub(/[^a-zA-Z0-9_\-\.]+/, '')
end
2014-10-08 09:44:25 -04:00
def visibility_level_field
:visibility_level
end
2014-10-08 09:44:25 -04:00
def embeddable?
Ability.allowed?(nil, :read_snippet, self)
end
def notes_with_associations
notes.includes(:author)
end
def check_for_spam?
visibility_level_changed?(to: Snippet::PUBLIC) ||
(public? && (title_changed? || content_changed?))
end
# snippets are the biggest sources of spam
override :allow_possible_spam?
def allow_possible_spam?
false
end
def spammable_entity_type
'snippet'
end
def to_ability_name
'snippet'
end
def valid_secret_token?(token)
return false unless token && secret_token
ActiveSupport::SecurityUtils.secure_compare(token.to_s, secret_token.to_s)
end
def as_json(options = {})
options[:except] = Array.wrap(options[:except])
options[:except] << :secret_token
super
end
override :repository
def repository
@repository ||= Repository.new(full_path, self, shard: repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::SNIPPET)
end
override :repository_size_checker
def repository_size_checker
strong_memoize(:repository_size_checker) do
::Gitlab::RepositorySizeChecker.new(
current_size_proc: -> { repository.size.megabytes },
limit: Gitlab::CurrentSettings.snippet_size_limit
)
end
end
override :storage
def storage
@storage ||= Storage::Hashed.new(self, prefix: Storage::Hashed::SNIPPET_REPOSITORY_PATH_PREFIX)
end
# This is the full_path used to identify the
# the snippet repository. It will be used mostly
# for logging purposes.
override :full_path
def full_path
return unless persisted?
@full_path ||= begin
components = []
components << project.full_path if project_id?
components << '@snippets'
components << self.id
components.join('/')
end
end
def repository_storage
snippet_repository&.shard_name || self.class.pick_repository_storage
end
def create_repository
return if repository_exists? && snippet_repository
repository.create_if_not_exists
track_snippet_repository(repository.storage)
end
def track_snippet_repository(shard)
snippet_repo = snippet_repository || build_snippet_repository
snippet_repo.update!(shard_name: shard, disk_path: disk_path)
end
def can_cache_field?(field)
field != :content || MarkupHelper.gitlab_markdown?(file_name)
end
def hexdigest
Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
end
def file_name_on_repo
return if repository.empty?
list_files(repository.root_ref).first
end
def list_files(ref = nil)
return [] if repository.empty?
repository.ls_files(ref)
end
def multiple_files?
list_files(repository.root_ref).size > 1
end
class << self
# Searches for snippets with a matching title, description or file name.
#
# This method uses ILIKE on PostgreSQL.
#
# query - The search query as a String.
#
# Returns an ActiveRecord::Relation.
def search(query)
fuzzy_search(query, [:title, :description, :file_name])
end
def parent_class
::Project
end
end
2011-10-16 17:07:10 -04:00
end
Snippet.prepend_if_ee('EE::Snippet')