kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

Browse source
This commit is contained in:
GitLab Bot 2020-03-06 18:08:08 +00:00
parent 57a37ce99f
commit 83731155d9
41 changed files with 521 additions and 378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitlab/ci/pages.gitlab-ci.yml
View file

@ -5,7 +5,7 @@ pages:
- .default-cache
- .pages:rules
stage: pages
dependencies: ["coverage", "karma", "gitlab:assets:compile pull-cache"]
dependencies: ["rspec:coverage", "karma", "gitlab:assets:compile pull-cache"]
script:
- mv public/ .public/
- mkdir public/

2
.gitlab/ci/rails.gitlab-ci.yml
View file

@ -202,7 +202,7 @@ gitlab:setup:
paths:
- log/development.log
coverage:
rspec:coverage:
extends:
- .rails-job-base
- .rails:rules:ee-and-foss

4
CHANGELOG-EE.md
View file

@ -1,5 +1,9 @@
Please view this file on the master branch, on stable branches it's out of date.
## 12.8.5
- No changes.
## 12.8.4
- No changes.

4
app/models/snippet.rb
View file

@ -302,6 +302,10 @@ class Snippet < ApplicationRecord
field != :content || MarkupHelper.gitlab_markdown?(file_name)
end
def hexdigest
Digest::SHA256.hexdigest("#{title}#{description}#{created_at}#{updated_at}")
end
class << self
# Searches for snippets with a matching title or file name.
#

6
app/models/snippet_repository.rb
View file

@ -18,6 +18,12 @@ class SnippetRepository < ApplicationRecord
end
end
def create_file(user, path, content, **options)
options[:actions] = transform_file_entries([{ file_path: path, content: content }])
capture_git_error { repository.multi_action(user, **options) }
end
def multi_files_action(user, files = [], **options)
return if files.nil? || files.empty?

6
app/services/projects/import_export/export_service.rb
View file

@ -42,7 +42,7 @@ module Projects
end
def exporters
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver]
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver]
end
def version_saver
@ -73,6 +73,10 @@ module Projects
Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared)
end
def snippets_repo_saver
Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: current_user, project: project, shared: shared)
end
def cleanup
FileUtils.rm_rf(shared.archive_path) if shared&.archive_path
end

4
app/views/layouts/nav/sidebar/_project.html.haml
View file

@ -143,7 +143,7 @@
- issue_tracker = @project.external_issue_tracker
= link_to issue_tracker.issue_tracker_path, class: 'shortcuts-external_tracker' do
.nav-icon-container
= sprite_icon('issue-external')
= sprite_icon('external-link')
%span.nav-item-name
= issue_tracker.title
%ul.sidebar-sub-level-items.is-fly-out-only
@ -319,7 +319,7 @@
= nav_link do
= link_to external_wiki_url, class: 'shortcuts-external_wiki' do
.nav-icon-container
= sprite_icon('issue-external')
= sprite_icon('external-link')
%span.nav-item-name
= _('External Wiki')
%ul.sidebar-sub-level-items.is-fly-out-only

6
changelogs/unreleased/198323-migrate-snippet-mentions-to-db-table.yml
View file

@ -1,6 +0,0 @@
---
title: Migrate mentions for snippet and snippet notes to snippet_user_mentions DB
table
merge_request: 23783
author:
type: changed

5
changelogs/unreleased/208827-replace-issue-external-icon-with-external-link.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Replace issue-external icon with external-link
merge_request: 208827
author:
type: other

5
changelogs/unreleased/fj-39201-import-export-project-snippets.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Import/Export snippet repositories
merge_request: 24150
author:
type: added

28
db/post_migrate/20200127111953_cleanup_empty_snippet_user_mentions.rb
View file

@ -1,28 +0,0 @@
# frozen_string_literal: true
class CleanupEmptySnippetUserMentions < ActiveRecord::Migration[5.2]
DOWNTIME = false
BATCH_SIZE = 10_000
class SnippetUserMention < ActiveRecord::Base
include EachBatch
self.table_name = 'snippet_user_mentions'
end
def up
# cleanup snippet user mentions with no actual mentions,
# re https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24586#note_285982468
SnippetUserMention
.where(mentioned_users_ids: nil)
.where(mentioned_groups_ids: nil)
.where(mentioned_projects_ids: nil)
.each_batch(of: BATCH_SIZE) do |batch|
batch.delete_all
end
end
def down
# no-op
end
end

35
db/post_migrate/20200127131953_migrate_snippet_mentions_to_db.rb
View file

@ -1,35 +0,0 @@
# frozen_string_literal: true
class MigrateSnippetMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
DELAY = 2.minutes.to_i
BATCH_SIZE = 10_000
MIGRATION = 'UserMentions::CreateResourceUserMention'
JOIN = "LEFT JOIN snippet_user_mentions on snippets.id = snippet_user_mentions.snippet_id"
QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND snippet_user_mentions.snippet_id IS NULL"
disable_ddl_transaction!
class Snippet < ActiveRecord::Base
include EachBatch
self.table_name = 'snippets'
end
def up
Snippet
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(snippets.id)'), Arel.sql('MAX(snippets.id)')).first
migrate_in(index * DELAY, MIGRATION, ['Snippet', JOIN, QUERY_CONDITIONS, false, *range])
end
end
def down
# no-op
end
end

20
db/post_migrate/20200127141953_add_temporary_snippet_notes_with_mentions_index.rb
View file

@ -1,20 +0,0 @@
# frozen_string_literal: true
class AddTemporarySnippetNotesWithMentionsIndex < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'snippet_mentions_temp_index'
INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Snippet'"
disable_ddl_transaction!
def up
# create temporary index for notes with mentions, may take well over 1h
add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
end
def down
remove_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
end
end

38
db/post_migrate/20200127151953_migrate_snippet_notes_mentions_to_db.rb
View file

@ -1,38 +0,0 @@
# frozen_string_literal: true
class MigrateSnippetNotesMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
DELAY = 2.minutes.to_i
BATCH_SIZE = 10_000
MIGRATION = 'UserMentions::CreateResourceUserMention'
INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Snippet'"
QUERY_CONDITIONS = "#{INDEX_CONDITION} AND snippet_user_mentions.snippet_id IS NULL"
JOIN = 'INNER JOIN snippets ON snippets.id = notes.noteable_id LEFT JOIN snippet_user_mentions ON notes.id = snippet_user_mentions.note_id'
disable_ddl_transaction!
class Note < ActiveRecord::Base
include EachBatch
self.table_name = 'notes'
end
def up
Note
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
migrate_in(index * DELAY, MIGRATION, ['Snippet', JOIN, QUERY_CONDITIONS, true, *range])
end
end
def down
# no-op
# temporary index is to be dropped in a different migration in an upcoming release:
# https://gitlab.com/gitlab-org/gitlab/issues/196842
end
end

1
db/schema.rb
View file

@ -2838,7 +2838,6 @@ ActiveRecord::Schema.define(version: 2020_03_04_160823) do
t.index ["discussion_id"], name: "index_notes_on_discussion_id"
t.index ["id"], name: "design_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'DesignManagement::Design'::text))"
t.index ["id"], name: "epic_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Epic'::text))"
t.index ["id"], name: "snippet_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Snippet'::text))"
t.index ["line_code"], name: "index_notes_on_line_code"
t.index ["note"], name: "index_notes_on_note_trigram", opclass: :gin_trgm_ops, using: :gin
t.index ["note"], name: "tmp_idx_on_promoted_notes", where: "(((noteable_type)::text = 'Issue'::text) AND (system IS TRUE) AND (note ~~ 'promoted to epic%'::text))"

40
doc/user/group/saml_sso/index.md
View file

@ -234,6 +234,29 @@ Recommended `NameID` value: `OneLogin ID`.
Set parameters according to the [assertions table](#assertions).
### Additional setup options
GitLab [isn't limited to the SAML providers listed above](#my-identity-provider-isnt-listed) but your Identity Provider may require additional configuration, such as the following:
| Field | Value | Notes |
|-------|-------|-------|
| SAML Profile | Web browser SSO profile | GitLab uses SAML to sign users in via their browser. We don't make requests direct to the Identity Provider. |
| SAML Request Binding | HTTP Redirect | GitLab (the service provider) redirects users to your Identity Provider with a base64 encoded `SAMLRequest` HTTP parameter. |
| SAML Response Binding | HTTP POST | Your Identity Provider responds to users with an HTTP form including the `SAMLResponse`, which a user's browser submits back to GitLab. |
| Sign SAML Response | Yes | We require this to prevent tampering. |
| X509 Certificate in response | Yes | This is used to sign the response and checked against the provided fingerprint. |
| Fingerprint Algorithm | SHA-1 | We need a SHA-1 hash of the certificate used to sign the SAML Response. |
| Signature Algorithm | SHA-1/SHA-256/SHA-384/SHA-512 | Also known as the Digest Method, this can be specified in the SAML response. It determines how a response is signed. |
| Encrypt SAML Assertion | No | TLS is used between your Identity Provider, the user's browser, and GitLab. |
| Sign SAML Assertion | Optional | We don't require Assertions to be signed. We validate their integrity by requiring the whole response to be signed. |
| Check SAML Request Signature | No | GitLab does not sign SAML requests, but does check the signature on the SAML response. |
| Default RelayState | Optional | The URL users should end up on after signing in via a button on your Identity Provider. |
| NameID Format | `Persistent` | See [details above](#nameid-format). |
| Additional URLs | | You may need to use the `Identifier` or `Assertion consumer service URL` in other fields on some providers. |
| Single Sign Out URL | | Not supported |
If the information information you need isn't listed above you may wish to check our [troubleshooting docs below](#i-need-additional-information-to-configure-my-identity-provider).
## Linking SAML to your existing GitLab.com account
To link SAML to your existing GitLab.com account:
@ -320,3 +343,20 @@ To change which identity you sign in with, you can [unlink the previous SAML ide
Getting both of these errors at the same time suggests the NameID capitalization provided by the Identity Provider didn't exactly match the previous value for that user.
This can be prevented by configuring the [NameID](#nameid) to return a consistent value. Fixing this for an individual user involves [unlinking SAML in the GitLab account](#unlinking-accounts), although this will cause group membership and Todos to be lost.
### My identity provider isn't listed
Not a problem, the SAML standard means that a wide range of identity providers will work with GitLab. Unfortunately we aren't familiar with all of them so can only offer support configuring the [listed providers](#providers).
### I need additional information to configure my identity provider
Many SAML terms can vary between providers. It is possible that the information you are looking for is listed under another name.
For more information, start with your Identity Provider's documentation. Look for their options and examples to see how they configure SAML. This can provide hints on what you'll need to configure GitLab to work with these providers.
It can also help to look at our [more detailed docs for self-managed GitLab](../../../integration/saml.md).
SAML configuration for GitLab.com is mostly the same as for self-managed instances.
However, self-managed GitLab instances use a configuration file that supports more options as described in the external [OmniAuth SAML documentation](https://github.com/omniauth/omniauth-saml/).
Internally that uses the [`ruby-saml` library](https://github.com/onelogin/ruby-saml), so we sometimes check there to verify low level details of less commonly used options.
It can also help to compare the XML response from your provider with our [example XML used for internal testing](https://gitlab.com/gitlab-org/gitlab/blob/master/ee/spec/fixtures/saml/response.xml).

1
doc/user/profile/account/two_factor_authentication.md
View file

@ -110,6 +110,7 @@ the **Download codes** button for storage in a safe place. If you choose to
download them, the file will be called `gitlab-recovery-codes.txt`.
If you lose the recovery codes or just want to generate new ones, you can do so
from the [two-factor authentication account settings page](#regenerate-2fa-recovery-codes) or
[using SSH](#generate-new-recovery-codes-using-ssh).
## Logging in with 2FA Enabled

2
lib/api/api.rb
View file

@ -45,7 +45,7 @@ module API
before do
Gitlab::ApplicationContext.push(
user: -> { current_user },
user: -> { @current_user },
project: -> { @project },
namespace: -> { @group },
caller_id: route.origin

43
lib/gitlab/background_migration/user_mentions/models/snippet.rb
View file

@ -1,43 +0,0 @@
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
module UserMentions
module Models
class Snippet < ActiveRecord::Base
include Concerns::IsolatedMentionable
include Concerns::MentionableMigrationMethods
include CacheMarkdownField
attr_mentionable :title, pipeline: :single_line
attr_mentionable :description
cache_markdown_field :title, pipeline: :single_line
cache_markdown_field :description
self.table_name = 'snippets'
self.inheritance_column = :_type_disabled
belongs_to :author, class_name: "User"
belongs_to :project
def self.user_mention_model
Gitlab::BackgroundMigration::UserMentions::Models::SnippetUserMention
end
def user_mention_model
self.class.user_mention_model
end
def user_mention_resource_id
id
end
def user_mention_note_id
'NULL'
end
end
end
end
end
end

18
lib/gitlab/background_migration/user_mentions/models/snippet_user_mention.rb
View file

@ -1,18 +0,0 @@
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
module UserMentions
module Models
class SnippetUserMention < ActiveRecord::Base
self.table_name = 'snippet_user_mentions'
def self.resource_foreign_key
:snippet_id
end
end
end
end
end
end

12
lib/gitlab/import_export.rb
View file

@ -42,6 +42,18 @@ module Gitlab
"project.wiki.bundle"
end
def snippet_repo_bundle_dir
'snippets'
end
def snippets_repo_bundle_path(absolute_path)
File.join(absolute_path, ::Gitlab::ImportExport.snippet_repo_bundle_dir)
end
def snippet_repo_bundle_filename_for(snippet)
"#{snippet.hexdigest}.bundle"
end
def config_file
Rails.root.join('lib/gitlab/import_export/project/import_export.yml')
end

8
lib/gitlab/import_export/importer.rb
View file

@ -35,7 +35,7 @@ module Gitlab
def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer, statistics_restorer]
uploads_restorer, lfs_restorer, statistics_restorer, snippets_repo_restorer]
end
def import_file
@ -79,6 +79,12 @@ module Gitlab
Gitlab::ImportExport::LfsRestorer.new(project: project, shared: shared)
end
def snippets_repo_restorer
Gitlab::ImportExport::SnippetsRepoRestorer.new(project: project,
shared: shared,
user: current_user)
end
def statistics_restorer
Gitlab::ImportExport::StatisticsRestorer.new(project: project, shared: shared)
end

48
lib/gitlab/import_export/snippet_repo_restorer.rb Normal file
View file

@ -0,0 +1,48 @@
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetRepoRestorer < RepoRestorer
attr_reader :snippet
def initialize(snippet:, user:, shared:, path_to_bundle:)
@snippet = snippet
@user = user
@repository = snippet.repository
@path_to_bundle = path_to_bundle.to_s
@shared = shared
end
def restore
if File.exist?(path_to_bundle)
create_repository_from_bundle
else
create_repository_from_db
end
true
rescue => e
shared.error(e)
false
end
private
def create_repository_from_bundle
repository.create_from_bundle(path_to_bundle)
snippet.track_snippet_repository
end
def create_repository_from_db
snippet.create_repository
commit_attrs = {
branch_name: 'master',
message: 'Initial commit'
}
repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs)
end
end
end
end

21
lib/gitlab/import_export/snippet_repo_saver.rb Normal file
View file

@ -0,0 +1,21 @@
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetRepoSaver < RepoSaver
def initialize(project:, shared:, repository:)
@project = project
@shared = shared
@repository = repository
end
private
def bundle_full_path
File.join(shared.export_path,
::Gitlab::ImportExport.snippet_repo_bundle_dir,
::Gitlab::ImportExport.snippet_repo_bundle_filename_for(repository.container))
end
end
end
end

36
lib/gitlab/import_export/snippets_repo_restorer.rb Normal file
View file

@ -0,0 +1,36 @@
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetsRepoRestorer
def initialize(project:, shared:, user:)
@project = project
@shared = shared
@user = user
end
def restore
return true unless Feature.enabled?(:version_snippets, @user)
return true unless Dir.exist?(snippets_repo_bundle_path)
@project.snippets.find_each.all? do |snippet|
Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet,
user: @user,
shared: @shared,
path_to_bundle: snippet_repo_bundle_path(snippet))
.restore
end
end
private
def snippet_repo_bundle_path(snippet)
File.join(snippets_repo_bundle_path, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet))
end
def snippets_repo_bundle_path
@snippets_repo_bundle_path ||= ::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path)
end
end
end
end

34
lib/gitlab/import_export/snippets_repo_saver.rb Normal file
View file

@ -0,0 +1,34 @@
# frozen_string_literal: true
module Gitlab
module ImportExport
class SnippetsRepoSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(current_user:, project:, shared:)
@project = project
@shared = shared
@current_user = current_user
end
def save
return true unless Feature.enabled?(:version_snippets, @current_user)
create_snippets_repo_directory
@project.snippets.find_each.all? do |snippet|
Gitlab::ImportExport::SnippetRepoSaver.new(project: @project,
shared: @shared,
repository: snippet.repository)
.save
end
end
private
def create_snippets_repo_directory
mkdir_p(::Gitlab::ImportExport.snippets_repo_bundle_path(@shared.export_path))
end
end
end
end

2
scripts/review_apps/automated_cleanup.rb
View file

@ -81,7 +81,7 @@ class AutomatedCleanup
release = Quality::HelmClient::Release.new(environment.slug, 1, deployed_at.to_s, nil, nil, review_apps_namespace)
releases_to_delete << release
end
elsif deployed_at < stop_threshold
elsif environment.state != 'stopped' && deployed_at < stop_threshold
stop_environment(environment, deployment)
else
print_release_state(subject: 'Review App', release_name: environment.slug, release_date: last_deploy, action: 'leaving')

2
spec/factories/snippets.rb
View file

@ -34,7 +34,7 @@ FactoryBot.define do
trait :empty_repo do
after(:create) do |snippet|
raise "Failed to create repository!" unless snippet.repository.create_if_not_exists
raise "Failed to create repository!" unless snippet.create_repository
end
end
end

77
spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb
View file

@ -1,77 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
require './db/post_migrate/20200127131953_migrate_snippet_mentions_to_db'
require './db/post_migrate/20200127151953_migrate_snippet_notes_mentions_to_db'
describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention, schema: 20200127151953 do
include MigrationsHelpers
context 'when migrating data' do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:notes) { table(:notes) }
let(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
let(:member) { users.create!(email: 'member@example.com', notification_email: 'member@example.com', name: 'member', username: 'member', projects_limit: 10, state: 'active') }
let(:admin) { users.create!(email: 'administrator@example.com', notification_email: 'administrator@example.com', name: 'administrator', username: 'administrator', admin: 1, projects_limit: 10, state: 'active') }
let(:john_doe) { users.create!(email: 'john_doe@example.com', notification_email: 'john_doe@example.com', name: 'john_doe', username: 'john_doe', projects_limit: 10, state: 'active') }
let(:skipped) { users.create!(email: 'skipped@example.com', notification_email: 'skipped@example.com', name: 'skipped', username: 'skipped', projects_limit: 10, state: 'active') }
let(:mentioned_users) { [author, member, admin, john_doe, skipped] }
let(:mentioned_users_refs) { mentioned_users.map { |u| "@#{u.username}" }.join(' ') }
let(:group) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') }
let(:inaccessible_group) { namespaces.create!(name: 'test2', path: 'test2', runners_token: 'my-token2', project_creation_level: 1, visibility_level: 0, type: 'Group') }
let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let(:mentioned_groups) { [group, inaccessible_group] }
let(:group_mentions) { [group, inaccessible_group].map { |gr| "@#{gr.path}" }.join(' ') }
let(:description_mentions) { "description with mentions #{mentioned_users_refs} and #{group_mentions}" }
before do
# build personal namespaces and routes for users
mentioned_users.each { |u| u.becomes(User).save! }
# build namespaces and routes for groups
mentioned_groups.each do |gr|
gr.name += '-org'
gr.path += '-org'
gr.becomes(Namespace).save!
end
end
context 'migrate snippet mentions' do
let(:snippets) { table(:snippets) }
let(:snippet_user_mentions) { table(:snippet_user_mentions) }
let!(:snippet1) { snippets.create!(project_id: project.id, author_id: author.id, title: 'title1', description: description_mentions) }
let!(:snippet2) { snippets.create!(project_id: project.id, author_id: author.id, title: 'title2', description: 'some description') }
let!(:snippet3) { snippets.create!(project_id: project.id, author_id: author.id, title: 'title3', description: 'description with an email@example.com and some other @ char here.') }
let(:user_mentions) { snippet_user_mentions }
let(:resource) { snippet1 }
it_behaves_like 'resource mentions migration', MigrateSnippetMentionsToDb, Snippet
context 'mentions in note' do
let!(:note1) { notes.create!(noteable_id: snippet1.id, noteable_type: 'Snippet', project_id: project.id, author_id: author.id, note: description_mentions) }
let!(:note2) { notes.create!(noteable_id: snippet1.id, noteable_type: 'Snippet', project_id: project.id, author_id: author.id, note: 'sample note') }
let!(:note3) { notes.create!(noteable_id: snippet1.id, noteable_type: 'Snippet', project_id: project.id, author_id: author.id, note: description_mentions, system: true) }
# this not does not have actual mentions
let!(:note4) { notes.create!(noteable_id: snippet1.id, noteable_type: 'Snippet', project_id: project.id, author_id: author.id, note: 'note3 for an email@somesite.com and some other rando @ ref' ) }
# this note points to an innexistent noteable record in snippets table
let!(:note5) { notes.create!(noteable_id: snippets.maximum(:id) + 10, noteable_type: 'Snippet', project_id: project.id, author_id: author.id, note: description_mentions) }
it_behaves_like 'resource notes mentions migration', MigrateSnippetNotesMentionsToDb, Snippet
end
end
end
context 'checks no_quote_columns' do
it 'has correct no_quote_columns' do
expect(Gitlab::BackgroundMigration::UserMentions::Models::Snippet.no_quote_columns).to match([:note_id, :snippet_id])
end
end
end

8
spec/lib/gitlab/import_export/import_export_spec.rb
View file

@ -21,4 +21,12 @@ describe Gitlab::ImportExport do
expect(described_class.export_filename(exportable: project).length).to be < 70
end
end
describe '#snippet_repo_bundle_filename_for' do
let(:snippet) { build(:snippet, id: 1) }
it 'generates the snippet bundle name' do
expect(described_class.snippet_repo_bundle_filename_for(snippet)).to eq "#{snippet.hexdigest}.bundle"
end
end
end

3
spec/lib/gitlab/import_export/importer_spec.rb
View file

@ -50,7 +50,8 @@ describe Gitlab::ImportExport::Importer do
Gitlab::ImportExport::WikiRestorer,
Gitlab::ImportExport::UploadsRestorer,
Gitlab::ImportExport::LfsRestorer,
Gitlab::ImportExport::StatisticsRestorer
Gitlab::ImportExport::StatisticsRestorer,
Gitlab::ImportExport::SnippetsRepoRestorer
].each do |restorer|
it "calls the #{restorer}" do
fake_restorer = double(restorer.to_s)

70
spec/lib/gitlab/import_export/snippet_repo_restorer_spec.rb Normal file
View file

@ -0,0 +1,70 @@
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetRepoRestorer do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, namespace: user.namespace) }
let(:snippet) { create(:project_snippet, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:exporter) { Gitlab::ImportExport::SnippetsRepoSaver.new(project: project, shared: shared, current_user: user) }
let(:restorer) do
described_class.new(user: user,
shared: shared,
snippet: snippet,
path_to_bundle: snippet_bundle_path)
end
after do
FileUtils.rm_rf(shared.export_path)
end
shared_examples 'no bundle file present' do
it 'creates the repository from the database content' do
expect(snippet.repository_exists?).to be_falsey
aggregate_failures do
expect(restorer.restore).to be_truthy
expect(snippet.repository_exists?).to be_truthy
expect(snippet.snippet_repository).not_to be_nil
blob = snippet.repository.blob_at('HEAD', snippet.file_name)
expect(blob).not_to be_nil
expect(blob.data).to eq(snippet.content)
end
end
end
context 'when the snippet does not have a bundle file path' do
let(:snippet_bundle_path) { nil }
it_behaves_like 'no bundle file present'
end
context 'when the snippet bundle path is not present' do
let(:snippet_bundle_path) { 'foo' }
it_behaves_like 'no bundle file present'
end
context 'when the snippet bundle exists' do
let!(:snippet_with_repo) { create(:project_snippet, :repository, project: project) }
let(:bundle_path) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
let(:snippet_bundle_path) { File.join(bundle_path, "#{snippet_with_repo.hexdigest}.bundle") }
let(:result) { exporter.save }
it 'creates the repository from the bundle' do
expect(exporter.save).to be_truthy
expect(snippet.repository_exists?).to be_falsey
expect(snippet.snippet_repository).to be_nil
expect(snippet.repository).to receive(:create_from_bundle).and_call_original
expect(restorer.restore).to be_truthy
expect(snippet.repository_exists?).to be_truthy
expect(snippet.snippet_repository).not_to be_nil
end
end
end

48
spec/lib/gitlab/import_export/snippet_repo_saver_spec.rb Normal file
View file

@ -0,0 +1,48 @@
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetRepoSaver do
describe 'bundle a project Git repo' do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, namespace: user.namespace) }
let_it_be(:snippet) { create(:project_snippet, :repository, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:bundler) { described_class.new(project: project, shared: shared, repository: snippet.repository) }
let(:bundle_path) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
around do |example|
FileUtils.mkdir_p(bundle_path)
example.run
ensure
FileUtils.rm_rf(bundle_path)
end
context 'with project snippet' do
it 'bundles the repo successfully' do
aggregate_failures do
expect(bundler.save).to be_truthy
expect(Dir.empty?(bundle_path)).to be_falsey
end
end
context 'when snippet does not have a repository' do
let(:snippet) { build(:personal_snippet) }
it 'returns true' do
expect(bundler.save).to be_truthy
end
it 'does not create any file' do
aggregate_failures do
expect(snippet.repository).not_to receive(:bundle_to_disk)
bundler.save
expect(Dir.empty?(bundle_path)).to be_truthy
end
end
end
end
end
end

55
spec/lib/gitlab/import_export/snippets_repo_restorer_spec.rb Normal file
View file

@ -0,0 +1,55 @@
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetsRepoRestorer do
include GitHelpers
describe 'bundle a snippet Git repo' do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, namespace: user.namespace) }
let_it_be(:snippet_with_repo) { create(:project_snippet, :repository, project: project, author: user) }
let_it_be(:snippet_without_repo) { create(:project_snippet, project: project, author: user) }
let(:shared) { project.import_export_shared }
let(:exporter) { Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: user, project: project, shared: shared) }
let(:bundle_dir) { ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path) }
let(:restorer) do
described_class.new(user: user,
shared: shared,
project: project)
end
let(:service) { instance_double(Gitlab::ImportExport::SnippetRepoRestorer) }
before do
exporter.save
end
after do
FileUtils.rm_rf(shared.export_path)
end
it 'calls SnippetRepoRestorer per each snippet with the bundle path' do
allow(service).to receive(:restore).and_return(true)
expect(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_with_repo, path_to_bundle: bundle_path(snippet_with_repo))).and_return(service)
expect(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_without_repo, path_to_bundle: bundle_path(snippet_without_repo))).and_return(service)
expect(restorer.restore).to be_truthy
end
context 'when one snippet cannot be saved' do
it 'returns false and do not process other snippets' do
allow(Gitlab::ImportExport::SnippetRepoRestorer).to receive(:new).with(hash_including(snippet: snippet_with_repo)).and_return(service)
allow(service).to receive(:restore).and_return(false)
expect(Gitlab::ImportExport::SnippetRepoRestorer).not_to receive(:new).with(hash_including(snippet: snippet_without_repo))
expect(restorer.restore).to be_falsey
end
end
def bundle_path(snippet)
File.join(bundle_dir, ::Gitlab::ImportExport.snippet_repo_bundle_filename_for(snippet))
end
end
end

56
spec/lib/gitlab/import_export/snippets_repo_saver_spec.rb Normal file
View file

@ -0,0 +1,56 @@
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::SnippetsRepoSaver do
describe 'bundle a project Git repo' do
let_it_be(:user) { create(:user) }
let!(:project) { create(:project) }
let(:shared) { project.import_export_shared }
let(:bundler) { described_class.new(current_user: user, project: project, shared: shared) }
after do
FileUtils.rm_rf(shared.export_path)
end
it 'creates the snippet bundles dir if not exists' do
snippets_dir = ::Gitlab::ImportExport.snippets_repo_bundle_path(shared.export_path)
expect(Dir.exist?(snippets_dir)).to be_falsey
bundler.save
expect(Dir.exist?(snippets_dir)).to be_truthy
end
context 'when project does not have any snippet' do
it 'does not perform any action' do
expect(Gitlab::ImportExport::SnippetRepoSaver).not_to receive(:new)
bundler.save
end
end
context 'when project has snippets' do
let!(:snippet1) { create(:project_snippet, :repository, project: project, author: user) }
let!(:snippet2) { create(:project_snippet, project: project, author: user) }
let(:service) { instance_double(Gitlab::ImportExport::SnippetRepoSaver) }
it 'calls the SnippetRepoSaver for each snippet' do
allow(Gitlab::ImportExport::SnippetRepoSaver).to receive(:new).and_return(service)
expect(service).to receive(:save).and_return(true).twice
bundler.save
end
context 'when one snippet cannot be saved' do
it 'returns false and do not process other snippets' do
allow(Gitlab::ImportExport::SnippetRepoSaver).to receive(:new).with(hash_including(repository: snippet1.repository)).and_return(service)
allow(service).to receive(:save).and_return(false)
expect(Gitlab::ImportExport::SnippetRepoSaver).not_to receive(:new).with(hash_including(repository: snippet2.repository))
expect(bundler.save).to be_falsey
end
end
end
end
end

42
spec/migrations/cleanup_empty_snippet_user_mentions_spec.rb
View file

@ -1,42 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200127111953_cleanup_empty_snippet_user_mentions')
describe CleanupEmptySnippetUserMentions, :migration, :sidekiq do
let(:users) { table(:users) }
let(:projects) { table(:projects) }
let(:namespaces) { table(:namespaces) }
let(:snippets) { table(:snippets) }
let(:snippet_user_mentions) { table(:snippet_user_mentions) }
let(:notes) { table(:notes) }
let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id) }
let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let(:snippet) { snippets.create!(title: "title1", title_html: 'title1', description: 'snippet description with @root mention', project_id: project.id, author_id: user.id) }
let!(:resource1) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet') }
let!(:resource2) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet', system: true) }
let!(:resource3) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet') }
# non-migrateable resources
# this note is already migrated, as it has a record in the snippet_user_mentions table
let!(:resource4) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet') }
let!(:user_mention) { snippet_user_mentions.create!(snippet_id: snippet.id, note_id: resource4.id, mentioned_users_ids: [1]) }
# this note points to an innexistent noteable record
let!(:resource5) { notes.create!(note: 'note for @root to check', noteable_id: snippets.maximum(:id) + 10, noteable_type: 'Snippet') }
# these should get cleanup, by the migration
let!(:blank_snippet_user_mention1) { snippet_user_mentions.create!(snippet_id: snippet.id, note_id: resource1.id)}
let!(:blank_snippet_user_mention2) { snippet_user_mentions.create!(snippet_id: snippet.id, note_id: resource2.id)}
let!(:blank_snippet_user_mention3) { snippet_user_mentions.create!(snippet_id: snippet.id, note_id: resource3.id)}
it 'cleanups blank user mentions' do
expect(snippet_user_mentions.count).to eq 4
migrate!
expect(snippet_user_mentions.count).to eq 1
end
end

28
spec/migrations/migrate_snippet_mentions_to_db_spec.rb
View file

@ -1,28 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200127131953_migrate_snippet_mentions_to_db')
describe MigrateSnippetMentionsToDb, :migration, :sidekiq do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:snippets) { table(:snippets) }
let(:snippet_user_mentions) { table(:snippet_user_mentions) }
let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id) }
let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let!(:resource1) { snippets.create!(title: "title1", title_html: 'title1', description: 'snippet description with @root mention', project_id: project.id, author_id: user.id) }
let!(:resource2) { snippets.create!(title: "title2", title_html: "title2", description: 'snippet description with @group mention', project_id: project.id, author_id: user.id) }
let!(:resource3) { snippets.create!(title: "title3", title_html: "title3", description: 'snippet description with @project mention', project_id: project.id, author_id: user.id) }
# non-migrateable resources
# this snippet is already migrated, as it has a record in the snippet_user_mentions table
let!(:resource4) { snippets.create!(title: "title4", title_html: "title4", description: 'snippet description with @project mention', project_id: project.id, author_id: user.id) }
let!(:user_mention) { snippet_user_mentions.create!(snippet_id: resource4.id, mentioned_users_ids: [1]) }
# this snippet has no mentions so should be filtered out
let!(:resource5) { snippets.create!(title: "title5", title_html: "title5", description: 'snippet description with no mention', project_id: project.id, author_id: user.id) }
it_behaves_like 'schedules resource mentions migration', Snippet, false
end

31
spec/migrations/migrate_snippet_notes_mentions_to_db_spec.rb
View file

@ -1,31 +0,0 @@
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200127151953_migrate_snippet_notes_mentions_to_db')
describe MigrateSnippetNotesMentionsToDb, :migration, :sidekiq do
let(:users) { table(:users) }
let(:projects) { table(:projects) }
let(:namespaces) { table(:namespaces) }
let(:snippets) { table(:snippets) }
let(:snippet_user_mentions) { table(:snippet_user_mentions) }
let(:notes) { table(:notes) }
let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) }
let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id) }
let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) }
let(:snippet) { snippets.create!(title: "title1", title_html: 'title1', description: 'snippet description with @root mention', project_id: project.id, author_id: user.id) }
let!(:resource1) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet') }
let!(:resource2) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet', system: true) }
let!(:resource3) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet') }
# non-migrateable resources
# this note is already migrated, as it has a record in the snippet_user_mentions table
let!(:resource4) { notes.create!(note: 'note for @root to check', noteable_id: snippet.id, noteable_type: 'Snippet') }
let!(:user_mention) { snippet_user_mentions.create!(snippet_id: snippet.id, note_id: resource4.id, mentioned_users_ids: [1]) }
# this note points to an innexistent noteable record
let!(:resource5) { notes.create!(note: 'note for @root to check', noteable_id: snippets.maximum(:id) + 10, noteable_type: 'Snippet') }
it_behaves_like 'schedules resource mentions migration', Snippet, true
end

38
spec/models/snippet_repository_spec.rb
View file

@ -26,6 +26,44 @@ describe SnippetRepository do
end
end
describe '#create_file' do
let(:snippet) { create(:personal_snippet, :empty_repo, author: user) }
it 'creates the file' do
snippet_repository.create_file(user, 'foo', 'bar', commit_opts)
blob = first_blob(snippet)
aggregate_failures do
expect(blob).not_to be_nil
expect(blob.path).to eq 'foo'
expect(blob.data).to eq 'bar'
end
end
it 'fills the file path if empty' do
snippet_repository.create_file(user, nil, 'bar', commit_opts)
blob = first_blob(snippet)
aggregate_failures do
expect(blob).not_to be_nil
expect(blob.path).to eq 'snippetfile1.txt'
expect(blob.data).to eq 'bar'
end
end
context 'when the file exists' do
let(:snippet) { create(:personal_snippet, :repository, author: user) }
it 'captures the git exception and raises a SnippetRepository::CommitError' do
existing_blob = first_blob(snippet)
expect do
snippet_repository.create_file(user, existing_blob.path, existing_blob.data, commit_opts)
end.to raise_error described_class::CommitError
end
end
end
describe '#multi_files_action' do
let(:new_file) { { file_path: 'new_file_test', content: 'bar' } }
let(:move_file) { { previous_path: 'CHANGELOG', file_path: 'CHANGELOG_new', content: 'bar' } }

8
spec/services/projects/import_export/export_service_spec.rb
View file

@ -64,6 +64,14 @@ describe Projects::ImportExport::ExportService do
service.execute
end
it 'saves the snippets' do
expect_next_instance_of(Gitlab::ImportExport::SnippetsRepoSaver) do |instance|
expect(instance).to receive(:save).and_call_original
end
service.execute
end
context 'when all saver services succeed' do
before do
allow(service).to receive(:save_services).and_return(true)

2
spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb
View file

@ -72,7 +72,7 @@ shared_examples 'schedules resource mentions migration' do |resource_class, is_f
it 'schedules background migrations' do
Sidekiq::Testing.fake! do
Timecop.freeze do
resource_count = is_for_notes ? Note.where(noteable_type: resource_class.to_s).count : resource_class.count
resource_count = is_for_notes ? Note.count : resource_class.count
expect(resource_count).to eq 5
migrate!