From 9a1203dfe866d46780b92e5cdb081dd6fdc778c5 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 7 Oct 2022 15:09:16 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .../javascripts/pages/projects/hooks/index.js | 2 + .../webhooks/components/form_url_app.vue | 73 ++++++++++++++++++ .../components/form_url_mask_item.vue | 61 +++++++++++++++ app/assets/javascripts/webhooks/index.js | 18 +++++ app/models/note.rb | 1 + app/views/shared/web_hooks/_form.html.haml | 13 ++-- app/workers/all_queues.yml | 18 +++++ .../attachments/import_note_worker.rb | 23 ++++++ .../attachments/import_release_worker.rb | 23 ++++++ .../import_release_attachments_worker.rb | 6 +- .../stage/import_attachments_worker.rb | 5 +- .../development/webhook_form_mask_url.yml | 8 ++ ...194453_add_index_releases_project_id_id.rb | 15 ++++ db/schema_migrations/20220929194453 | 1 + db/structure.sql | 2 + doc/development/github_importer.md | 10 ++- doc/user/project/import/github.md | 5 ++ lib/gitlab/github_import/exceptions.rb | 11 +++ .../importer/attachments/base_importer.rb | 51 +++++++++++++ .../importer/attachments/notes_importer.rb | 35 +++++++++ .../importer/attachments/releases_importer.rb | 33 +++++++++ ...porter.rb => note_attachments_importer.rb} | 30 +++++--- .../github_import/importer/note_importer.rb | 3 + .../importer/releases_attachments_importer.rb | 59 --------------- .../importer/releases_importer.rb | 3 + .../github_import/representation/note_text.rb | 60 +++++++++++++++ .../representation/release_attachments.rb | 44 ----------- locale/gitlab.pot | 18 +++++ .../settings/webhooks_settings_spec.rb | 8 +- .../webhooks/components/form_url_app_spec.js | 53 +++++++++++++ .../components/form_url_mask_item_spec.js | 51 +++++++++++++ .../attachments/base_importer_spec.rb | 28 +++++++ .../attachments/notes_importer_spec.rb | 66 +++++++++++++++++ .../attachments/releases_importer_spec.rb | 65 ++++++++++++++++ .../note_attachments_importer_spec.rb | 68 +++++++++++++++++ .../importer/note_importer_spec.rb | 7 ++ .../release_attachments_importer_spec.rb | 57 -------------- .../releases_attachments_importer_spec.rb | 74 ------------------- .../representation/note_text_spec.rb | 64 ++++++++++++++++ .../release_attachments_spec.rb | 49 ------------ spec/workers/every_sidekiq_worker_spec.rb | 2 + .../attachments/import_note_worker_spec.rb | 49 ++++++++++++ .../attachments/import_release_worker_spec.rb | 49 ++++++++++++ .../import_release_attachments_worker_spec.rb | 6 +- .../stage/import_attachments_worker_spec.rb | 23 ++++-- 45 files changed, 1030 insertions(+), 320 deletions(-) create mode 100644 app/assets/javascripts/webhooks/components/form_url_app.vue create mode 100644 app/assets/javascripts/webhooks/components/form_url_mask_item.vue create mode 100644 app/assets/javascripts/webhooks/index.js create mode 100644 app/workers/gitlab/github_import/attachments/import_note_worker.rb create mode 100644 app/workers/gitlab/github_import/attachments/import_release_worker.rb create mode 100644 config/feature_flags/development/webhook_form_mask_url.yml create mode 100644 db/post_migrate/20220929194453_add_index_releases_project_id_id.rb create mode 100644 db/schema_migrations/20220929194453 create mode 100644 lib/gitlab/github_import/exceptions.rb create mode 100644 lib/gitlab/github_import/importer/attachments/base_importer.rb create mode 100644 lib/gitlab/github_import/importer/attachments/notes_importer.rb create mode 100644 lib/gitlab/github_import/importer/attachments/releases_importer.rb rename lib/gitlab/github_import/importer/{release_attachments_importer.rb => note_attachments_importer.rb} (63%) delete mode 100644 lib/gitlab/github_import/importer/releases_attachments_importer.rb create mode 100644 lib/gitlab/github_import/representation/note_text.rb delete mode 100644 lib/gitlab/github_import/representation/release_attachments.rb create mode 100644 spec/frontend/webhooks/components/form_url_app_spec.js create mode 100644 spec/frontend/webhooks/components/form_url_mask_item_spec.js create mode 100644 spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb create mode 100644 spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb create mode 100644 spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb create mode 100644 spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb delete mode 100644 spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb delete mode 100644 spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb create mode 100644 spec/lib/gitlab/github_import/representation/note_text_spec.rb delete mode 100644 spec/lib/gitlab/github_import/representation/release_attachments_spec.rb create mode 100644 spec/workers/gitlab/github_import/attachments/import_note_worker_spec.rb create mode 100644 spec/workers/gitlab/github_import/attachments/import_release_worker_spec.rb diff --git a/app/assets/javascripts/pages/projects/hooks/index.js b/app/assets/javascripts/pages/projects/hooks/index.js index 2a120a690ef..ed476d25f8b 100644 --- a/app/assets/javascripts/pages/projects/hooks/index.js +++ b/app/assets/javascripts/pages/projects/hooks/index.js @@ -1,3 +1,5 @@ import initSearchSettings from '~/search_settings'; +import initWebhookForm from '~/webhooks'; initSearchSettings(); +initWebhookForm(); diff --git a/app/assets/javascripts/webhooks/components/form_url_app.vue b/app/assets/javascripts/webhooks/components/form_url_app.vue new file mode 100644 index 00000000000..62d6c03bbb3 --- /dev/null +++ b/app/assets/javascripts/webhooks/components/form_url_app.vue @@ -0,0 +1,73 @@ + + + diff --git a/app/assets/javascripts/webhooks/components/form_url_mask_item.vue b/app/assets/javascripts/webhooks/components/form_url_mask_item.vue new file mode 100644 index 00000000000..1e74b4a8215 --- /dev/null +++ b/app/assets/javascripts/webhooks/components/form_url_mask_item.vue @@ -0,0 +1,61 @@ + + + diff --git a/app/assets/javascripts/webhooks/index.js b/app/assets/javascripts/webhooks/index.js new file mode 100644 index 00000000000..bfa33560fa5 --- /dev/null +++ b/app/assets/javascripts/webhooks/index.js @@ -0,0 +1,18 @@ +import Vue from 'vue'; +import FormUrlApp from './components/form_url_app.vue'; + +export default () => { + const el = document.querySelector('.js-vue-webhook-form'); + + if (!el) { + return null; + } + + return new Vue({ + el, + name: 'WebhookFormRoot', + render(createElement) { + return createElement(FormUrlApp, {}); + }, + }); +}; diff --git a/app/models/note.rb b/app/models/note.rb index 6375d3f777f..e444111119b 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -22,6 +22,7 @@ class Note < ApplicationRecord include ThrottledTouch include FromUnion include Sortable + include EachBatch ISSUE_TASK_SYSTEM_NOTE_PATTERN = /\A.*marked\sthe\stask.+as\s(completed|incomplete).*\z/.freeze diff --git a/app/views/shared/web_hooks/_form.html.haml b/app/views/shared/web_hooks/_form.html.haml index afe72767b9a..549436ccabf 100644 --- a/app/views/shared/web_hooks/_form.html.haml +++ b/app/views/shared/web_hooks/_form.html.haml @@ -1,10 +1,13 @@ = form_errors(hook) -.form-group - = form.label :url, s_('Webhooks|URL'), class: 'label-bold' - = form.text_field :url, class: 'form-control gl-form-input', placeholder: 'http://example.com/trigger-ci.json' - %p.form-text.text-muted - = s_('Webhooks|URL must be percent-encoded if it contains one or more special characters.') +- if Feature.enabled?(:webhook_form_mask_url) + .js-vue-webhook-form +- else + .form-group + = form.label :url, s_('Webhooks|URL'), class: 'label-bold' + = form.text_field :url, class: 'form-control gl-form-input', placeholder: 'http://example.com/trigger-ci.json' + %p.form-text.text-muted + = s_('Webhooks|URL must be percent-encoded if it contains one or more special characters.') .form-group = form.label :token, s_('Webhooks|Secret token'), class: 'label-bold' = form.text_field :token, class: 'form-control gl-form-input', placeholder: '' diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index e02285e7f34..fcd37c43cb3 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1020,6 +1020,24 @@ :weight: 1 :idempotent: false :tags: [] +- :name: github_importer:github_import_attachments_import_note + :worker_name: Gitlab::GithubImport::Attachments::ImportNoteWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] +- :name: github_importer:github_import_attachments_import_release + :worker_name: Gitlab::GithubImport::Attachments::ImportReleaseWorker + :feature_category: :importers + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: false + :tags: [] - :name: github_importer:github_import_import_diff_note :worker_name: Gitlab::GithubImport::ImportDiffNoteWorker :feature_category: :importers diff --git a/app/workers/gitlab/github_import/attachments/import_note_worker.rb b/app/workers/gitlab/github_import/attachments/import_note_worker.rb new file mode 100644 index 00000000000..2f5bc50ee0b --- /dev/null +++ b/app/workers/gitlab/github_import/attachments/import_note_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Attachments + class ImportNoteWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::NoteText + end + + def importer_class + Importer::NoteAttachmentsImporter + end + + def object_type + :note_attachment + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/attachments/import_release_worker.rb b/app/workers/gitlab/github_import/attachments/import_release_worker.rb new file mode 100644 index 00000000000..5eea5702d3c --- /dev/null +++ b/app/workers/gitlab/github_import/attachments/import_release_worker.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Attachments + class ImportReleaseWorker # rubocop:disable Scalability/IdempotentWorker + include ObjectImporter + + def representation_class + Representation::NoteText + end + + def importer_class + Importer::NoteAttachmentsImporter + end + + def object_type + :release_attachment + end + end + end + end +end diff --git a/app/workers/gitlab/github_import/import_release_attachments_worker.rb b/app/workers/gitlab/github_import/import_release_attachments_worker.rb index c6f45ec1d7c..bf901f2f7b8 100644 --- a/app/workers/gitlab/github_import/import_release_attachments_worker.rb +++ b/app/workers/gitlab/github_import/import_release_attachments_worker.rb @@ -1,16 +1,18 @@ # frozen_string_literal: true +# TODO: remove in 16.X milestone +# https://gitlab.com/gitlab-org/gitlab/-/issues/377059 module Gitlab module GithubImport class ImportReleaseAttachmentsWorker # rubocop:disable Scalability/IdempotentWorker include ObjectImporter def representation_class - Representation::ReleaseAttachments + Representation::NoteText end def importer_class - Importer::ReleaseAttachmentsImporter + Importer::NoteAttachmentsImporter end def object_type diff --git a/app/workers/gitlab/github_import/stage/import_attachments_worker.rb b/app/workers/gitlab/github_import/stage/import_attachments_worker.rb index e9086dca503..727de632df3 100644 --- a/app/workers/gitlab/github_import/stage/import_attachments_worker.rb +++ b/app/workers/gitlab/github_import/stage/import_attachments_worker.rb @@ -28,7 +28,10 @@ module Gitlab # For future issue/mr/milestone/etc attachments importers def importers - [::Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter] + [ + ::Gitlab::GithubImport::Importer::Attachments::ReleasesImporter, + ::Gitlab::GithubImport::Importer::Attachments::NotesImporter + ] end def start_importer(project, importer, client) diff --git a/config/feature_flags/development/webhook_form_mask_url.yml b/config/feature_flags/development/webhook_form_mask_url.yml new file mode 100644 index 00000000000..445fcb0b6b3 --- /dev/null +++ b/config/feature_flags/development/webhook_form_mask_url.yml @@ -0,0 +1,8 @@ +--- +name: webhook_form_mask_url +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/99995 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/376106 +milestone: '15.5' +type: development +group: group::integrations +default_enabled: false diff --git a/db/post_migrate/20220929194453_add_index_releases_project_id_id.rb b/db/post_migrate/20220929194453_add_index_releases_project_id_id.rb new file mode 100644 index 00000000000..179b5a636f0 --- /dev/null +++ b/db/post_migrate/20220929194453_add_index_releases_project_id_id.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddIndexReleasesProjectIdId < Gitlab::Database::Migration[2.0] + INDEX_NAME = 'index_releases_on_project_id_id' + + disable_ddl_transaction! + + def up + add_concurrent_index :releases, %i[project_id id], name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :releases, name: INDEX_NAME + end +end diff --git a/db/schema_migrations/20220929194453 b/db/schema_migrations/20220929194453 new file mode 100644 index 00000000000..25d512b792b --- /dev/null +++ b/db/schema_migrations/20220929194453 @@ -0,0 +1 @@ +0f625bd9df16d035dd8cd73d5097e5b5f4d1b900183bd6ccf4b1489319535df6 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a01cca4893b..6360fe3b797 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -30160,6 +30160,8 @@ CREATE INDEX index_releases_on_author_id_id_created_at ON releases USING btree ( CREATE INDEX index_releases_on_project_id_and_released_at_and_id ON releases USING btree (project_id, released_at, id); +CREATE INDEX index_releases_on_project_id_id ON releases USING btree (project_id, id); + CREATE UNIQUE INDEX index_releases_on_project_tag_unique ON releases USING btree (project_id, tag); CREATE INDEX index_releases_on_released_at ON releases USING btree (released_at); diff --git a/doc/development/github_importer.md b/doc/development/github_importer.md index c2dd6c89544..af787633b06 100644 --- a/doc/development/github_importer.md +++ b/doc/development/github_importer.md @@ -127,13 +127,15 @@ comments. ### 10. Stage::ImportAttachmentsWorker -This worker imports release notes attachments that are linked inside Markdown. -For every release of the project, we schedule a job of -`Gitlab::GithubImport::ImportReleaseAttachmentsWorker` for every comment. +This worker imports note attachments that are linked inside Markdown. +For each entity with Markdown text in the project, we schedule a job of: + +- `Gitlab::GithubImport::ImportReleaseAttachmentsWorker` for every release. +- `Gitlab::GithubImport::ImportNoteAttachmentsWorker` for every note. Each job: -1. Iterates over all attachment links inside of a specific release note. +1. Iterates over all attachment links inside of a specific record. 1. Downloads the attachment. 1. Replaces the old link with a newly-generated link to GitLab. diff --git a/doc/user/project/import/github.md b/doc/user/project/import/github.md index 0ae6e828086..76a3e71e339 100644 --- a/doc/user/project/import/github.md +++ b/doc/user/project/import/github.md @@ -178,6 +178,11 @@ The following items of a project are imported: - Milestones. - Labels. - Release note descriptions. +- Attachments for: + - Release notes. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15620) in GitLab 15.4. + - Comments and notes. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/18052) in GitLab 15.5. + + NOTE: All attachment importers work under `github_importer_attachments_import` [feature flag](../../../administration/feature_flags.md) disabled by default. - Release note attachments. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15620) in GitLab 15.4 with `github_importer_attachments_import` [feature flag](../../../administration/feature_flags.md) disabled by default. - Pull request review comments. diff --git a/lib/gitlab/github_import/exceptions.rb b/lib/gitlab/github_import/exceptions.rb new file mode 100644 index 00000000000..3a36b64a11b --- /dev/null +++ b/lib/gitlab/github_import/exceptions.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Exceptions + # Sometimes it's not clear which of not implemented interfaces caused this error. + # We need custom exception to be able to add text that gives extra context. + NotImplementedError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/base_importer.rb b/lib/gitlab/github_import/importer/attachments/base_importer.rb new file mode 100644 index 00000000000..52983872a22 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/base_importer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class BaseImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + collection.each_batch(of: BATCH_SIZE) do |batch| + batch.each do |record| + next if already_imported?(record) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield record + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(record) + end + end + end + + def representation_class + Representation::NoteText + end + + def importer_class + NoteAttachmentsImporter + end + + private + + def collection + raise Gitlab::GithubImport::Exceptions::NotImplementedError, '#collection' + end + + def object_representation(object) + representation_class.from_db_record(object) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/notes_importer.rb b/lib/gitlab/github_import/importer/attachments/notes_importer.rb new file mode 100644 index 00000000000..aa38a7a3a3f --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/notes_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class NotesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportNoteWorker + end + + def collection_method + :note_attachments + end + + def object_type + :note_attachment + end + + def id_for_already_imported_cache(note) + note.id + end + + private + + # TODO: exclude :system, :noteable_type from select after removing override Note#note method + # https://gitlab.com/gitlab-org/gitlab/-/issues/369923 + def collection + project.notes.user.select(:id, :note, :system, :noteable_type) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb new file mode 100644 index 00000000000..feaa69eff71 --- /dev/null +++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Attachments + class ReleasesImporter < ::Gitlab::GithubImport::Importer::Attachments::BaseImporter + def sidekiq_worker_class + ::Gitlab::GithubImport::Attachments::ImportReleaseWorker + end + + def collection_method + :release_attachments + end + + def object_type + :release_attachment + end + + def id_for_already_imported_cache(release) + release.id + end + + private + + def collection + project.releases.select(:id, :description) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/release_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb similarity index 63% rename from lib/gitlab/github_import/importer/release_attachments_importer.rb rename to lib/gitlab/github_import/importer/note_attachments_importer.rb index 6419851623c..c0ea7e109ef 100644 --- a/lib/gitlab/github_import/importer/release_attachments_importer.rb +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -3,26 +3,27 @@ module Gitlab module GithubImport module Importer - class ReleaseAttachmentsImporter - attr_reader :release_db_id, :release_description, :project + class NoteAttachmentsImporter + attr_reader :note_text, :project - # release - An instance of `ReleaseAttachments`. + # note_text - An instance of `NoteText`. # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. - def initialize(release_attachments, project, _client = nil) - @release_db_id = release_attachments.release_db_id - @release_description = release_attachments.description + def initialize(note_text, project, _client = nil) + @note_text = note_text @project = project end def execute - attachment_urls = MarkdownText.fetch_attachment_urls(release_description) - new_description = attachment_urls.reduce(release_description) do |description, url| + attachment_urls = MarkdownText.fetch_attachment_urls(note_text.text) + return if attachment_urls.blank? + + new_text = attachment_urls.reduce(note_text.text) do |text, url| new_url = download_attachment(url) - description.gsub(url, new_url) + text.gsub(url, new_url) end - Release.find(release_db_id).update_column(:description, new_description) + update_note_record(new_text) end private @@ -52,6 +53,15 @@ module Gitlab def extract_name_from_markdown(text) text.match(%r{^!?\[.*\]}).to_a[0] end + + def update_note_record(text) + case note_text.record_type + when ::Release.name + ::Release.find(note_text.record_db_id).update_column(:description, text) + when ::Note.name + ::Note.find(note_text.record_db_id).update_column(:note, text) + end + end end end end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 1410006af26..69b7b2c2a38 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -36,6 +36,9 @@ module Gitlab # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. + # Note: if you're going to replace `legacy_bulk_insert` with something that trigger callback + # to generate HTML version - you also need to regenerate it in + # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since diff --git a/lib/gitlab/github_import/importer/releases_attachments_importer.rb b/lib/gitlab/github_import/importer/releases_attachments_importer.rb deleted file mode 100644 index 7221c802d83..00000000000 --- a/lib/gitlab/github_import/importer/releases_attachments_importer.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class ReleasesAttachmentsImporter - include ParallelScheduling - - BATCH_SIZE = 100 - - # The method that will be called for traversing through all the objects to - # import, yielding them to the supplied block. - def each_object_to_import - project.releases.select(:id, :description).each_batch(of: BATCH_SIZE, column: :id) do |batch| - batch.each do |release| - next if already_imported?(release) - - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - - yield release - - # We mark the object as imported immediately so we don't end up - # scheduling it multiple times. - mark_as_imported(release) - end - end - end - - def representation_class - Representation::ReleaseAttachments - end - - def importer_class - ReleaseAttachmentsImporter - end - - def sidekiq_worker_class - ImportReleaseAttachmentsWorker - end - - def collection_method - :release_attachments - end - - def object_type - :release_attachment - end - - def id_for_already_imported_cache(release) - release.id - end - - def object_representation(object) - representation_class.from_db_record(object) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 07efd22b0fe..fe6da30bbf8 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -12,6 +12,9 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # Note: if you're going to replace `legacy_bulk_insert` with something that triggers callback + # to generate HTML version - you also need to regenerate it in + # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. def execute bulk_insert(Release, build_releases) end diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb new file mode 100644 index 00000000000..1dae32fc79c --- /dev/null +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# This class only partly represents Release record from DB and +# is used to connect ReleasesAttachmentsImporter, NotesAttachmentsImporter etc. +# with NoteAttachmentsImporter without modifying ObjectImporter a lot. +# Attachments are inside release's `description`. +module Gitlab + module GithubImport + module Representation + class NoteText + include ToHash + include ExposeAttribute + + MODELS_WHITELIST = [::Release, ::Note].freeze + ModelNotSupported = Class.new(StandardError) + + attr_reader :attributes + + expose_attribute :record_db_id, :record_type, :text + + class << self + # Builds a note text representation from DB record of Note or Release. + # + # record - An instance of `Release` or `Note` model. + def from_db_record(record) + check_record_class!(record) + + record_type = record.class.name + text = record.is_a?(Release) ? record.description : record.note + new( + record_db_id: record.id, + record_type: record_type, + text: text + ) + end + + def from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + private + + def check_record_class!(record) + raise ModelNotSupported, record.class.name if MODELS_WHITELIST.exclude?(record.class) + end + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { db_id: record_db_id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/release_attachments.rb b/lib/gitlab/github_import/representation/release_attachments.rb deleted file mode 100644 index fd272be2405..00000000000 --- a/lib/gitlab/github_import/representation/release_attachments.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -# This class only partly represents Release record from DB and -# is used to connect ReleasesAttachmentsImporter with ReleaseAttachmentsImporter -# without modifying ObjectImporter a lot. -# Attachments are inside release's `description`. -module Gitlab - module GithubImport - module Representation - class ReleaseAttachments - include ToHash - include ExposeAttribute - - attr_reader :attributes - - expose_attribute :release_db_id, :description - - # Builds a event from a GitHub API response. - # - # release - An instance of `Release` model. - def self.from_db_record(release) - new( - release_db_id: release.id, - description: release.description - ) - end - - def self.from_json_hash(raw_hash) - new Representation.symbolize_hash(raw_hash) - end - - # attributes - A Hash containing the event details. The keys of this - # Hash (and any nested hashes) must be symbols. - def initialize(attributes) - @attributes = attributes - end - - def github_identifiers - { db_id: release_db_id } - end - end - end - end -end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 7cfe9c67b0d..c10e9eaba39 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -45011,6 +45011,9 @@ msgstr "" msgid "Webhooks|Deployment events" msgstr "" +msgid "Webhooks|Do not show sensitive data such as tokens in the UI." +msgstr "" + msgid "Webhooks|Enable SSL verification" msgstr "" @@ -45026,12 +45029,18 @@ msgstr "" msgid "Webhooks|Go to webhooks" msgstr "" +msgid "Webhooks|How it looks in the UI" +msgstr "" + msgid "Webhooks|Issues events" msgstr "" msgid "Webhooks|Job events" msgstr "" +msgid "Webhooks|Mask portions of URL" +msgstr "" + msgid "Webhooks|Member events" msgstr "" @@ -45056,6 +45065,12 @@ msgstr "" msgid "Webhooks|Secret token" msgstr "" +msgid "Webhooks|Sensitive portion of URL" +msgstr "" + +msgid "Webhooks|Show full URL" +msgstr "" + msgid "Webhooks|Subgroup events" msgstr "" @@ -45080,6 +45095,9 @@ msgstr "" msgid "Webhooks|URL must be percent-encoded if it contains one or more special characters." msgstr "" +msgid "Webhooks|URL preview" +msgstr "" + msgid "Webhooks|Used to validate received payloads. Sent with the request in the %{code_start}X-Gitlab-Token HTTP%{code_end} header." msgstr "" diff --git a/spec/features/projects/settings/webhooks_settings_spec.rb b/spec/features/projects/settings/webhooks_settings_spec.rb index d525544ac15..25752bcaf45 100644 --- a/spec/features/projects/settings/webhooks_settings_spec.rb +++ b/spec/features/projects/settings/webhooks_settings_spec.rb @@ -48,10 +48,10 @@ RSpec.describe 'Projects > Settings > Webhook Settings' do expect(page).to have_content('Releases events') end - it 'create webhook' do + it 'create webhook', :js do visit webhooks_path - fill_in 'hook_url', with: url + fill_in 'URL', with: url check 'Tag push events' fill_in 'hook_push_events_branch_filter', with: 'master' check 'Enable SSL verification' @@ -66,12 +66,12 @@ RSpec.describe 'Projects > Settings > Webhook Settings' do expect(page).to have_content('Job events') end - it 'edit existing webhook' do + it 'edit existing webhook', :js do hook visit webhooks_path click_link 'Edit' - fill_in 'hook_url', with: url + fill_in 'URL', with: url check 'Enable SSL verification' click_button 'Save changes' diff --git a/spec/frontend/webhooks/components/form_url_app_spec.js b/spec/frontend/webhooks/components/form_url_app_spec.js new file mode 100644 index 00000000000..40de3cc0d33 --- /dev/null +++ b/spec/frontend/webhooks/components/form_url_app_spec.js @@ -0,0 +1,53 @@ +import { nextTick } from 'vue'; +import { GlFormRadio, GlFormRadioGroup } from '@gitlab/ui'; + +import FormUrlApp from '~/webhooks/components/form_url_app.vue'; + +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; + +describe('FormUrlApp', () => { + let wrapper; + + const createComponent = () => { + wrapper = shallowMountExtended(FormUrlApp); + }; + + afterEach(() => { + wrapper.destroy(); + }); + + const findAllRadioButtons = () => wrapper.findAllComponents(GlFormRadio); + const findRadioGroup = () => wrapper.findComponent(GlFormRadioGroup); + const findUrlMaskDisable = () => findAllRadioButtons().at(0); + const findUrlMaskEnable = () => findAllRadioButtons().at(1); + const findUrlMaskSection = () => wrapper.findByTestId('url-mask-section'); + + describe('template', () => { + it('renders radio buttons for URL masking', () => { + createComponent(); + + expect(findAllRadioButtons().length).toBe(2); + expect(findUrlMaskDisable().text()).toBe(FormUrlApp.i18n.radioFullUrlText); + expect(findUrlMaskEnable().text()).toBe(FormUrlApp.i18n.radioMaskUrlText); + }); + + it('does not render mask section', () => { + createComponent(); + + expect(findUrlMaskSection().exists()).toBe(false); + }); + + describe('on radio select', () => { + beforeEach(async () => { + createComponent(); + + findRadioGroup().vm.$emit('input', true); + await nextTick(); + }); + + it('renders mask section', () => { + expect(findUrlMaskSection().exists()).toBe(true); + }); + }); + }); +}); diff --git a/spec/frontend/webhooks/components/form_url_mask_item_spec.js b/spec/frontend/webhooks/components/form_url_mask_item_spec.js new file mode 100644 index 00000000000..76681e6ab26 --- /dev/null +++ b/spec/frontend/webhooks/components/form_url_mask_item_spec.js @@ -0,0 +1,51 @@ +import { GlButton, GlFormInput } from '@gitlab/ui'; + +import FormUrlMaskItem from '~/webhooks/components/form_url_mask_item.vue'; + +import { shallowMountExtended } from 'helpers/vue_test_utils_helper'; + +describe('FormUrlMaskItem', () => { + let wrapper; + + const defaultProps = { + index: 0, + }; + + const createComponent = () => { + wrapper = shallowMountExtended(FormUrlMaskItem, { + propsData: { ...defaultProps }, + }); + }; + + afterEach(() => { + wrapper.destroy(); + }); + + const findMaskItemKey = () => wrapper.findByTestId('mask-item-key'); + const findMaskItemValue = () => wrapper.findByTestId('mask-item-value'); + const findRemoveButton = () => wrapper.findComponent(GlButton); + + describe('template', () => { + it('renders input for key and value', () => { + createComponent(); + + const keyInput = findMaskItemKey(); + expect(keyInput.attributes('label')).toBe(FormUrlMaskItem.i18n.keyLabel); + expect(keyInput.findComponent(GlFormInput).attributes('name')).toBe( + 'hook[url_variables][][key]', + ); + + const valueInput = findMaskItemValue(); + expect(valueInput.attributes('label')).toBe(FormUrlMaskItem.i18n.valueLabel); + expect(valueInput.findComponent(GlFormInput).attributes('name')).toBe( + 'hook[url_variables][][value]', + ); + }); + + it('renders remove button', () => { + createComponent(); + + expect(findRemoveButton().props('icon')).toBe('remove'); + }); + }); +}); diff --git a/spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb new file mode 100644 index 00000000000..5e60be44621 --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/base_importer_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::BaseImporter do + subject(:importer) { importer_class.new(project, client) } + + let(:project) { instance_double(Project, id: 1) } + let(:client) { instance_double(Gitlab::GithubImport::Client) } + let(:importer_class) do + Class.new(described_class) do + private + + def collection_method + 'test' + end + end + end + + describe '#each_object_to_import' do + context 'with not implemented #collection interface' do + it 'raises NotImplementedError' do + expect { importer.each_object_to_import } + .to raise_error(Gitlab::GithubImport::Exceptions::NotImplementedError, '#collection') + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb new file mode 100644 index 00000000000..f66a7b2eb8f --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/notes_importer_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::NotesImporter do + subject(:importer) { described_class.new(project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + describe '#sequential_import', :clean_gitlab_redis_cache do + let_it_be(:note_1) { create(:note, project: project) } + let_it_be(:note_2) { create(:note, project: project) } + let_it_be(:system_note) { create(:note, :system, project: project) } + + let(:importer_stub) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } + + it 'imports each project user note' do + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).twice.and_return(importer_stub) + expect(importer_stub).to receive(:execute).twice + + importer.sequential_import + end + + context 'when note is already processed' do + it "doesn't import this note" do + importer.mark_as_imported(note_1) + + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).once.and_return(importer_stub) + expect(importer_stub).to receive(:execute).once + + importer.sequential_import + end + end + end + + describe '#representation_class' do + it { expect(importer.representation_class).to eq(Gitlab::GithubImport::Representation::NoteText) } + end + + describe '#importer_class' do + it { expect(importer.importer_class).to eq(Gitlab::GithubImport::Importer::NoteAttachmentsImporter) } + end + + describe '#sidekiq_worker_class' do + it { expect(importer.sidekiq_worker_class).to eq(Gitlab::GithubImport::Attachments::ImportNoteWorker) } + end + + describe '#collection_method' do + it { expect(importer.collection_method).to eq(:note_attachments) } + end + + describe '#object_type' do + it { expect(importer.object_type).to eq(:note_attachment) } + end + + describe '#id_for_already_imported_cache' do + let(:note) { build_stubbed(:note) } + + it { expect(importer.id_for_already_imported_cache(note)).to eq(note.id) } + end +end diff --git a/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb new file mode 100644 index 00000000000..584e6301f4c --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/attachments/releases_importer_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::Attachments::ReleasesImporter do + subject(:importer) { described_class.new(project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + describe '#sequential_import', :clean_gitlab_redis_cache do + let_it_be(:release_1) { create(:release, project: project) } + let_it_be(:release_2) { create(:release, project: project) } + + let(:importer_stub) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + let(:importer_attrs) { [instance_of(Gitlab::GithubImport::Representation::NoteText), project, client] } + + it 'imports each project release' do + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).twice.and_return(importer_stub) + expect(importer_stub).to receive(:execute).twice + + importer.sequential_import + end + + context 'when note is already processed' do + it "doesn't import this release" do + importer.mark_as_imported(release_1) + + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter).to receive(:new) + .with(*importer_attrs).once.and_return(importer_stub) + expect(importer_stub).to receive(:execute).once + + importer.sequential_import + end + end + end + + describe '#representation_class' do + it { expect(importer.representation_class).to eq(Gitlab::GithubImport::Representation::NoteText) } + end + + describe '#importer_class' do + it { expect(importer.importer_class).to eq(Gitlab::GithubImport::Importer::NoteAttachmentsImporter) } + end + + describe '#sidekiq_worker_class' do + it { expect(importer.sidekiq_worker_class).to eq(Gitlab::GithubImport::Attachments::ImportReleaseWorker) } + end + + describe '#collection_method' do + it { expect(importer.collection_method).to eq(:release_attachments) } + end + + describe '#object_type' do + it { expect(importer.object_type).to eq(:release_attachment) } + end + + describe '#id_for_already_imported_cache' do + let(:release) { build_stubbed(:release) } + + it { expect(importer.id_for_already_imported_cache(release)).to eq(release.id) } + end +end diff --git a/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb new file mode 100644 index 00000000000..e4d5ea77dcf --- /dev/null +++ b/spec/lib/gitlab/github_import/importer/note_attachments_importer_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::NoteAttachmentsImporter do + subject(:importer) { described_class.new(note_text, project, client) } + + let_it_be(:project) { create(:project) } + + let(:client) { instance_double('Gitlab::GithubImport::Client') } + + let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } + let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } + let(:text) do + <<-TEXT.strip + Some text... + + [special-doc](#{doc_url}) + ![image.jpeg](#{image_url}) + TEXT + end + + describe '#execute' do + let(:downloader_stub) { instance_double(Gitlab::GithubImport::AttachmentsDownloader) } + let(:tmp_stub_doc) { Tempfile.create('attachment_download_test.txt') } + let(:tmp_stub_image) { Tempfile.create('image.jpeg') } + + before do + allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(doc_url) + .and_return(downloader_stub) + allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(image_url) + .and_return(downloader_stub) + allow(downloader_stub).to receive(:perform).and_return(tmp_stub_doc, tmp_stub_image) + allow(downloader_stub).to receive(:delete).twice + + allow(UploadService).to receive(:new) + .with(project, tmp_stub_doc, FileUploader).and_call_original + allow(UploadService).to receive(:new) + .with(project, tmp_stub_image, FileUploader).and_call_original + end + + context 'when importing release attachments' do + let(:release) { create(:release, project: project, description: text) } + let(:note_text) { Gitlab::GithubImport::Representation::NoteText.from_db_record(release) } + + it 'updates release description with new attachment urls' do + importer.execute + + release.reload + expect(release.description).to start_with("Some text...\n\n [special-doc](/uploads/") + expect(release.description).to include('![image.jpeg](/uploads/') + end + end + + context 'when importing note attachments' do + let(:note) { create(:note, project: project, note: text) } + let(:note_text) { Gitlab::GithubImport::Representation::NoteText.from_db_record(note) } + + it 'updates note text with new attachment urls' do + importer.execute + + note.reload + expect(note.note).to start_with("Some text...\n\n [special-doc](/uploads/") + expect(note.note).to include('![image.jpeg](/uploads/') + end + end + end +end diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb index 165f543525d..c60ecd85e92 100644 --- a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -160,6 +160,13 @@ RSpec.describe Gitlab::GithubImport::Importer::NoteImporter do expect(project.notes.take).to be_valid end + + # rubocop:disable RSpec/AnyInstanceOf + it 'skips markdown field cache callback' do + expect_any_instance_of(Note).not_to receive(:refresh_markdown_cache) + importer.execute + end + # rubocop:enable RSpec/AnyInstanceOf end describe '#find_noteable_id' do diff --git a/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb deleted file mode 100644 index 4779f9c8982..00000000000 --- a/spec/lib/gitlab/github_import/importer/release_attachments_importer_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter do - subject(:importer) { described_class.new(release_attachments, project, client) } - - let_it_be(:project) { create(:project) } - - let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:release) { create(:release, project: project, description: description) } - let(:release_attachments) do - Gitlab::GithubImport::Representation::ReleaseAttachments - .from_json_hash(release_db_id: release.id, description: release.description) - end - - let(:doc_url) { 'https://github.com/nickname/public-test-repo/files/9020437/git-cheat-sheet.txt' } - let(:image_url) { 'https://user-images.githubusercontent.com/6833842/0cf366b61ef2.jpeg' } - let(:description) do - <<-TEXT.strip - Some text... - - [special-doc](#{doc_url}) - ![image.jpeg](#{image_url}) - TEXT - end - - describe '#execute' do - let(:downloader_stub) { instance_double(Gitlab::GithubImport::AttachmentsDownloader) } - let(:tmp_stub_doc) { Tempfile.create('attachment_download_test.txt') } - let(:tmp_stub_image) { Tempfile.create('image.jpeg') } - - context 'when importing doc attachment' do - before do - allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(doc_url) - .and_return(downloader_stub) - allow(Gitlab::GithubImport::AttachmentsDownloader).to receive(:new).with(image_url) - .and_return(downloader_stub) - allow(downloader_stub).to receive(:perform).and_return(tmp_stub_doc, tmp_stub_image) - allow(downloader_stub).to receive(:delete).twice - - allow(UploadService).to receive(:new) - .with(project, tmp_stub_doc, FileUploader).and_call_original - allow(UploadService).to receive(:new) - .with(project, tmp_stub_image, FileUploader).and_call_original - end - - it 'updates release description with new attachment url' do - importer.execute - - release.reload - expect(release.description).to start_with("Some text...\n\n [special-doc](/uploads/") - expect(release.description).to include('![image.jpeg](/uploads/') - end - end - end -end diff --git a/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb b/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb deleted file mode 100644 index 1aeb3462cd5..00000000000 --- a/spec/lib/gitlab/github_import/importer/releases_attachments_importer_spec.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter do - subject { described_class.new(project, client) } - - let_it_be(:project) { create(:project) } - - let(:client) { instance_double(Gitlab::GithubImport::Client) } - - describe '#each_object_to_import', :clean_gitlab_redis_cache do - let!(:release_1) { create(:release, project: project) } - let!(:release_2) { create(:release, project: project) } - - it 'iterates each project release' do - list = [] - subject.each_object_to_import do |object| - list << object - end - expect(list).to contain_exactly(release_1, release_2) - end - - context 'when release is already processed' do - it "doesn't process this release" do - subject.mark_as_imported(release_1) - - list = [] - subject.each_object_to_import do |object| - list << object - end - expect(list).to contain_exactly(release_2) - end - end - end - - describe '#representation_class' do - it { expect(subject.representation_class).to eq(Gitlab::GithubImport::Representation::ReleaseAttachments) } - end - - describe '#importer_class' do - it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter) } - end - - describe '#sidekiq_worker_class' do - it { expect(subject.sidekiq_worker_class).to eq(Gitlab::GithubImport::ImportReleaseAttachmentsWorker) } - end - - describe '#collection_method' do - it { expect(subject.collection_method).to eq(:release_attachments) } - end - - describe '#object_type' do - it { expect(subject.object_type).to eq(:release_attachment) } - end - - describe '#id_for_already_imported_cache' do - let(:release) { build_stubbed(:release) } - - it { expect(subject.id_for_already_imported_cache(release)).to eq(release.id) } - end - - describe '#object_representation' do - let(:release) { build_stubbed(:release) } - - it 'returns release attachments representation' do - representation = subject.object_representation(release) - - expect(representation.class).to eq subject.representation_class - expect(representation.release_db_id).to eq release.id - expect(representation.description).to eq release.description - end - end -end diff --git a/spec/lib/gitlab/github_import/representation/note_text_spec.rb b/spec/lib/gitlab/github_import/representation/note_text_spec.rb new file mode 100644 index 00000000000..adced223032 --- /dev/null +++ b/spec/lib/gitlab/github_import/representation/note_text_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Representation::NoteText do + shared_examples 'a Note text data' do |match_record_type| + it 'returns an instance of ReleaseAttachments' do + expect(representation).to be_an_instance_of(described_class) + end + + it 'includes record DB id' do + expect(representation.record_db_id).to eq 42 + end + + it 'includes record type' do + expect(representation.record_type).to eq match_record_type + end + + it 'includes note text' do + expect(representation.text).to eq 'Some text here..' + end + end + + describe '.from_db_record' do + context 'with Release' do + let(:record) { build_stubbed(:release, id: 42, description: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Release' do + let(:representation) { described_class.from_db_record(record) } + end + end + + context 'with Note' do + let(:record) { build_stubbed(:note, id: 42, note: 'Some text here..') } + + it_behaves_like 'a Note text data', 'Note' do + let(:representation) { described_class.from_db_record(record) } + end + end + end + + describe '.from_json_hash' do + it_behaves_like 'a Note text data', 'Release' do + let(:hash) do + { + 'record_db_id' => 42, + 'record_type' => 'Release', + 'text' => 'Some text here..' + } + end + + let(:representation) { described_class.from_json_hash(hash) } + end + end + + describe '#github_identifiers' do + it 'returns a hash with needed identifiers' do + record_id = rand(100) + representation = described_class.new(record_db_id: record_id, text: 'text') + + expect(representation.github_identifiers).to eq({ db_id: record_id }) + end + end +end diff --git a/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb b/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb deleted file mode 100644 index 0ef9dad6a13..00000000000 --- a/spec/lib/gitlab/github_import/representation/release_attachments_spec.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::GithubImport::Representation::ReleaseAttachments do - shared_examples 'a Release attachments data' do - it 'returns an instance of ReleaseAttachments' do - expect(representation).to be_an_instance_of(described_class) - end - - it 'includes release DB id' do - expect(representation.release_db_id).to eq 42 - end - - it 'includes release description' do - expect(representation.description).to eq 'Some text here..' - end - end - - describe '.from_db_record' do - let(:release) { build_stubbed(:release, id: 42, description: 'Some text here..') } - - it_behaves_like 'a Release attachments data' do - let(:representation) { described_class.from_db_record(release) } - end - end - - describe '.from_json_hash' do - it_behaves_like 'a Release attachments data' do - let(:hash) do - { - 'release_db_id' => 42, - 'description' => 'Some text here..' - } - end - - let(:representation) { described_class.from_json_hash(hash) } - end - end - - describe '#github_identifiers' do - it 'returns a hash with needed identifiers' do - release_id = rand(100) - representation = described_class.new(release_db_id: release_id, description: 'text') - - expect(representation.github_identifiers).to eq({ db_id: release_id }) - end - end -end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index 708f937dea3..2f652ac6472 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -258,6 +258,8 @@ RSpec.describe 'Every Sidekiq worker' do 'GitGarbageCollectWorker' => false, 'Gitlab::GithubImport::AdvanceStageWorker' => 3, 'Gitlab::GithubImport::ImportReleaseAttachmentsWorker' => 5, + 'Gitlab::GithubImport::Attachments::ImportReleaseWorker' => 5, + 'Gitlab::GithubImport::Attachments::ImportNoteWorker' => 5, 'Gitlab::GithubImport::ImportDiffNoteWorker' => 5, 'Gitlab::GithubImport::ImportIssueWorker' => 5, 'Gitlab::GithubImport::ImportIssueEventWorker' => 5, diff --git a/spec/workers/gitlab/github_import/attachments/import_note_worker_spec.rb b/spec/workers/gitlab/github_import/attachments/import_note_worker_spec.rb new file mode 100644 index 00000000000..7b60cdecca6 --- /dev/null +++ b/spec/workers/gitlab/github_import/attachments/import_note_worker_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Attachments::ImportNoteWorker do + subject(:worker) { described_class.new } + + describe '#import' do + let(:import_state) { create(:import_state, :started) } + + let(:project) do + instance_double('Project', full_path: 'foo/bar', id: 1, import_state: import_state) + end + + let(:client) { instance_double('Gitlab::GithubImport::Client') } + let(:importer) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + + let(:note_hash) do + { + 'record_db_id' => rand(100), + 'record_type' => 'Note', + 'text' => <<-TEXT + Some text... + + ![special-image](https://user-images.githubusercontent.com...) + TEXT + } + end + + it 'imports an release attachments' do + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::NoteText), + project, + client + ) + .and_return(importer) + + expect(importer).to receive(:execute) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .and_call_original + + worker.import(project, client, note_hash) + end + end +end diff --git a/spec/workers/gitlab/github_import/attachments/import_release_worker_spec.rb b/spec/workers/gitlab/github_import/attachments/import_release_worker_spec.rb new file mode 100644 index 00000000000..e49b2fb6504 --- /dev/null +++ b/spec/workers/gitlab/github_import/attachments/import_release_worker_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Attachments::ImportReleaseWorker do + subject(:worker) { described_class.new } + + describe '#import' do + let(:import_state) { create(:import_state, :started) } + + let(:project) do + instance_double('Project', full_path: 'foo/bar', id: 1, import_state: import_state) + end + + let(:client) { instance_double('Gitlab::GithubImport::Client') } + let(:importer) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } + + let(:note_hash) do + { + 'record_db_id' => rand(100), + 'record_type' => 'Release', + 'text' => <<-TEXT + Some text... + + ![special-image](https://user-images.githubusercontent.com...) + TEXT + } + end + + it 'imports an release attachments' do + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter) + .to receive(:new) + .with( + an_instance_of(Gitlab::GithubImport::Representation::NoteText), + project, + client + ) + .and_return(importer) + + expect(importer).to receive(:execute) + + expect(Gitlab::GithubImport::ObjectCounter) + .to receive(:increment) + .and_call_original + + worker.import(project, client, note_hash) + end + end +end diff --git a/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb b/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb index cd53c6ee9c0..1d32d5c0e21 100644 --- a/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_release_attachments_worker_spec.rb @@ -13,7 +13,7 @@ RSpec.describe Gitlab::GithubImport::ImportReleaseAttachmentsWorker do end let(:client) { instance_double('Gitlab::GithubImport::Client') } - let(:importer) { instance_double('Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter') } + let(:importer) { instance_double('Gitlab::GithubImport::Importer::NoteAttachmentsImporter') } let(:release_hash) do { @@ -27,10 +27,10 @@ RSpec.describe Gitlab::GithubImport::ImportReleaseAttachmentsWorker do end it 'imports an issue event' do - expect(Gitlab::GithubImport::Importer::ReleaseAttachmentsImporter) + expect(Gitlab::GithubImport::Importer::NoteAttachmentsImporter) .to receive(:new) .with( - an_instance_of(Gitlab::GithubImport::Representation::ReleaseAttachments), + an_instance_of(Gitlab::GithubImport::Representation::NoteText), project, client ) diff --git a/spec/workers/gitlab/github_import/stage/import_attachments_worker_spec.rb b/spec/workers/gitlab/github_import/stage/import_attachments_worker_spec.rb index c2c5e1dbf4e..015a2beed7e 100644 --- a/spec/workers/gitlab/github_import/stage/import_attachments_worker_spec.rb +++ b/spec/workers/gitlab/github_import/stage/import_attachments_worker_spec.rb @@ -10,26 +10,32 @@ RSpec.describe Gitlab::GithubImport::Stage::ImportAttachmentsWorker do let(:feature_flag_state) { [group] } describe '#import' do - let(:importer) { instance_double('Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter') } + let(:releases_importer) { instance_double('Gitlab::GithubImport::Importer::Attachments::ReleasesImporter') } + let(:notes_importer) { instance_double('Gitlab::GithubImport::Importer::Attachments::NotesImporter') } let(:client) { instance_double('Gitlab::GithubImport::Client') } + let(:releases_waiter) { Gitlab::JobWaiter.new(2, '123') } + let(:notes_waiter) { Gitlab::JobWaiter.new(3, '234') } before do stub_feature_flags(github_importer_attachments_import: feature_flag_state) end it 'imports release attachments' do - waiter = Gitlab::JobWaiter.new(2, '123') - - expect(Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter) + expect(Gitlab::GithubImport::Importer::Attachments::ReleasesImporter) .to receive(:new) .with(project, client) - .and_return(importer) + .and_return(releases_importer) + expect(releases_importer).to receive(:execute).and_return(releases_waiter) - expect(importer).to receive(:execute).and_return(waiter) + expect(Gitlab::GithubImport::Importer::Attachments::NotesImporter) + .to receive(:new) + .with(project, client) + .and_return(notes_importer) + expect(notes_importer).to receive(:execute).and_return(notes_waiter) expect(Gitlab::GithubImport::AdvanceStageWorker) .to receive(:perform_async) - .with(project.id, { '123' => 2 }, :protected_branches) + .with(project.id, { '123' => 2, '234' => 3 }, :protected_branches) worker.import(client, project) end @@ -38,7 +44,8 @@ RSpec.describe Gitlab::GithubImport::Stage::ImportAttachmentsWorker do let(:feature_flag_state) { false } it 'skips release attachments import and calls next stage' do - expect(Gitlab::GithubImport::Importer::ReleasesAttachmentsImporter).not_to receive(:new) + expect(Gitlab::GithubImport::Importer::Attachments::ReleasesImporter).not_to receive(:new) + expect(Gitlab::GithubImport::Importer::Attachments::NotesImporter).not_to receive(:new) expect(Gitlab::GithubImport::AdvanceStageWorker) .to receive(:perform_async).with(project.id, {}, :protected_branches)