Merge branch 'elastic-test-repos-ce' into 'master'
Backport of elastic-test-repos from EE See merge request gitlab-org/gitlab-ce!24200
This commit is contained in:
Nick Thomas
commit
74dca275d2
3 changed files with 137 additions and 74 deletions
|
|
@ -1,94 +1,136 @@
|
|||
require './spec/support/sidekiq'
|
||||
|
||||
# rubocop:disable Rails/Output
|
||||
|
||||
Sidekiq::Testing.inline! do
|
||||
Gitlab::Seeder.quiet do
|
||||
project_urls = [
|
||||
'https://gitlab.com/gitlab-org/gitlab-test.git',
|
||||
'https://gitlab.com/gitlab-org/gitlab-shell.git',
|
||||
'https://gitlab.com/gnuwget/wget2.git',
|
||||
'https://gitlab.com/Commit451/LabCoat.git',
|
||||
'https://github.com/jashkenas/underscore.git',
|
||||
'https://github.com/flightjs/flight.git',
|
||||
'https://github.com/twitter/typeahead.js.git',
|
||||
'https://github.com/h5bp/html5-boilerplate.git',
|
||||
'https://github.com/google/material-design-lite.git',
|
||||
'https://github.com/jlevy/the-art-of-command-line.git',
|
||||
'https://github.com/FreeCodeCamp/freecodecamp.git',
|
||||
'https://github.com/google/deepdream.git',
|
||||
'https://github.com/jtleek/datasharing.git',
|
||||
'https://github.com/WebAssembly/design.git',
|
||||
'https://github.com/airbnb/javascript.git',
|
||||
'https://github.com/tessalt/echo-chamber-js.git',
|
||||
'https://github.com/atom/atom.git',
|
||||
'https://github.com/mattermost/mattermost-server.git',
|
||||
'https://github.com/purifycss/purifycss.git',
|
||||
'https://github.com/facebook/nuclide.git',
|
||||
'https://github.com/wbkd/awesome-d3.git',
|
||||
'https://github.com/kilimchoi/engineering-blogs.git',
|
||||
'https://github.com/gilbarbara/logos.git',
|
||||
'https://github.com/reduxjs/redux.git',
|
||||
'https://github.com/awslabs/s2n.git',
|
||||
'https://github.com/arkency/reactjs_koans.git',
|
||||
'https://github.com/twbs/bootstrap.git',
|
||||
'https://github.com/chjj/ttystudio.git',
|
||||
'https://github.com/MostlyAdequate/mostly-adequate-guide.git',
|
||||
'https://github.com/octocat/Spoon-Knife.git',
|
||||
'https://github.com/opencontainers/runc.git',
|
||||
'https://github.com/googlesamples/android-topeka.git'
|
||||
]
|
||||
Gitlab::Seeder.without_gitaly_timeout do
|
||||
project_urls = %w[
|
||||
https://gitlab.com/gitlab-org/gitlab-test.git
|
||||
https://gitlab.com/gitlab-org/gitlab-shell.git
|
||||
https://gitlab.com/gnuwget/wget2.git
|
||||
https://gitlab.com/Commit451/LabCoat.git
|
||||
https://github.com/jashkenas/underscore.git
|
||||
https://github.com/flightjs/flight.git
|
||||
https://github.com/twitter/typeahead.js.git
|
||||
https://github.com/h5bp/html5-boilerplate.git
|
||||
https://github.com/google/material-design-lite.git
|
||||
https://github.com/jlevy/the-art-of-command-line.git
|
||||
https://github.com/FreeCodeCamp/freecodecamp.git
|
||||
https://github.com/google/deepdream.git
|
||||
https://github.com/jtleek/datasharing.git
|
||||
https://github.com/WebAssembly/design.git
|
||||
https://github.com/airbnb/javascript.git
|
||||
https://github.com/tessalt/echo-chamber-js.git
|
||||
https://github.com/atom/atom.git
|
||||
https://github.com/mattermost/mattermost-server.git
|
||||
https://github.com/purifycss/purifycss.git
|
||||
https://github.com/facebook/nuclide.git
|
||||
https://github.com/wbkd/awesome-d3.git
|
||||
https://github.com/kilimchoi/engineering-blogs.git
|
||||
https://github.com/gilbarbara/logos.git
|
||||
https://github.com/reduxjs/redux.git
|
||||
https://github.com/awslabs/s2n.git
|
||||
https://github.com/arkency/reactjs_koans.git
|
||||
https://github.com/twbs/bootstrap.git
|
||||
https://github.com/chjj/ttystudio.git
|
||||
https://github.com/MostlyAdequate/mostly-adequate-guide.git
|
||||
https://github.com/octocat/Spoon-Knife.git
|
||||
https://github.com/opencontainers/runc.git
|
||||
https://github.com/googlesamples/android-topeka.git
|
||||
]
|
||||
|
||||
# You can specify how many projects you need during seed execution
|
||||
size = ENV['SIZE'].present? ? ENV['SIZE'].to_i : 8
|
||||
large_project_urls = %w[
|
||||
https://github.com/torvalds/linux.git
|
||||
https://gitlab.gnome.org/GNOME/gimp.git
|
||||
https://gitlab.gnome.org/GNOME/gnome-mud.git
|
||||
https://gitlab.com/fdroid/fdroidclient.git
|
||||
https://gitlab.com/inkscape/inkscape.git
|
||||
https://github.com/gnachman/iTerm2.git
|
||||
]
|
||||
|
||||
project_urls.first(size).each_with_index do |url, i|
|
||||
group_path, project_path = url.split('/')[-2..-1]
|
||||
def create_project(url, force_latest_storage: false)
|
||||
group_path, project_path = url.split('/')[-2..-1]
|
||||
|
||||
group = Group.find_by(path: group_path)
|
||||
group = Group.find_by(path: group_path)
|
||||
|
||||
unless group
|
||||
group = Group.new(
|
||||
name: group_path.titleize,
|
||||
path: group_path
|
||||
)
|
||||
group.description = FFaker::Lorem.sentence
|
||||
group.save
|
||||
unless group
|
||||
group = Group.new(
|
||||
name: group_path.titleize,
|
||||
path: group_path
|
||||
)
|
||||
group.description = FFaker::Lorem.sentence
|
||||
group.save
|
||||
|
||||
group.add_owner(User.first)
|
||||
group.add_owner(User.first)
|
||||
end
|
||||
|
||||
project_path.gsub!(".git", "")
|
||||
|
||||
params = {
|
||||
import_url: url,
|
||||
namespace_id: group.id,
|
||||
name: project_path.titleize,
|
||||
description: FFaker::Lorem.sentence,
|
||||
visibility_level: Gitlab::VisibilityLevel.values.sample,
|
||||
skip_disk_validation: true
|
||||
}
|
||||
|
||||
if force_latest_storage
|
||||
params[:storage_version] = Project::LATEST_STORAGE_VERSION
|
||||
end
|
||||
|
||||
project = nil
|
||||
|
||||
Sidekiq::Worker.skipping_transaction_check do
|
||||
project = Projects::CreateService.new(User.first, params).execute
|
||||
|
||||
# Seed-Fu runs this entire fixture in a transaction, so the `after_commit`
|
||||
# hook won't run until after the fixture is loaded. That is too late
|
||||
# since the Sidekiq::Testing block has already exited. Force clearing
|
||||
# the `after_commit` queue to ensure the job is run now.
|
||||
project.send(:_run_after_commit_queue)
|
||||
project.import_state.send(:_run_after_commit_queue)
|
||||
end
|
||||
|
||||
if project.valid? && project.valid_repo?
|
||||
print '.'
|
||||
else
|
||||
puts project.errors.full_messages
|
||||
print 'F'
|
||||
end
|
||||
end
|
||||
|
||||
project_path.gsub!(".git", "")
|
||||
# You can specify how many projects you need during seed execution
|
||||
size = ENV['SIZE'].present? ? ENV['SIZE'].to_i : 8
|
||||
|
||||
params = {
|
||||
import_url: url,
|
||||
namespace_id: group.id,
|
||||
name: project_path.titleize,
|
||||
description: FFaker::Lorem.sentence,
|
||||
visibility_level: Gitlab::VisibilityLevel.values.sample,
|
||||
skip_disk_validation: true
|
||||
}
|
||||
|
||||
if i % 2 == 0
|
||||
params[:storage_version] = Project::LATEST_STORAGE_VERSION
|
||||
project_urls.first(size).each_with_index do |url, i|
|
||||
create_project(url, force_latest_storage: i.even?)
|
||||
end
|
||||
|
||||
project = nil
|
||||
if ENV['LARGE_PROJECTS'].present?
|
||||
large_project_urls.each(&method(:create_project))
|
||||
|
||||
Sidekiq::Worker.skipping_transaction_check do
|
||||
project = Projects::CreateService.new(User.first, params).execute
|
||||
if ENV['FORK'].present?
|
||||
puts "\nGenerating forks"
|
||||
|
||||
# Seed-Fu runs this entire fixture in a transaction, so the `after_commit`
|
||||
# hook won't run until after the fixture is loaded. That is too late
|
||||
# since the Sidekiq::Testing block has already exited. Force clearing
|
||||
# the `after_commit` queue to ensure the job is run now.
|
||||
project.send(:_run_after_commit_queue)
|
||||
project.import_state.send(:_run_after_commit_queue)
|
||||
end
|
||||
project_name = ENV['FORK'] == 'true' ? 'torvalds/linux' : ENV['FORK']
|
||||
|
||||
if project.valid? && project.valid_repo?
|
||||
print '.'
|
||||
else
|
||||
puts project.errors.full_messages
|
||||
print 'F'
|
||||
project = Project.find_by_full_path(project_name)
|
||||
|
||||
User.offset(1).first(5).each do |user|
|
||||
new_project = Projects::ForkService.new(project, user).execute
|
||||
|
||||
if new_project.valid? && (new_project.valid_repo? || new_project.import_state.scheduled?)
|
||||
print '.'
|
||||
else
|
||||
new_project.errors.full_messages.each do |error|
|
||||
puts "#{new_project.full_path}: #{error}"
|
||||
end
|
||||
print 'F'
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -38,6 +38,14 @@ Note that since you can't see the questions from stdout, you might just want
|
|||
to `echo 'yes'` to keep it running. It would still print the errors on stderr
|
||||
so no worries about missing errors.
|
||||
|
||||
### Extra Project seed options
|
||||
|
||||
There are a few environment flags you can pass to change how projects are seeded
|
||||
|
||||
- `SIZE`: defaults to `8`, max: `32`. Amount of projects to create.
|
||||
- `LARGE_PROJECTS`: defaults to false. If set will clone 6 large projects to help with testing.
|
||||
- `FORK`: defaults to false. If set to `true` will fork `torvalds/linux` five times. Can also be set to an existing project full_path and it will fork that instead.
|
||||
|
||||
### Notes for MySQL
|
||||
|
||||
Since the seeds would contain various UTF-8 characters, such as emojis or so,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,19 @@ module Gitlab
|
|||
puts "\nOK".color(:green)
|
||||
end
|
||||
|
||||
def self.without_gitaly_timeout
|
||||
# Remove Gitaly timeout
|
||||
old_timeout = Gitlab::CurrentSettings.current_application_settings.gitaly_timeout_default
|
||||
Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: 0)
|
||||
# Otherwise we still see the default value when running seed_fu
|
||||
ApplicationSetting.expire
|
||||
|
||||
yield
|
||||
ensure
|
||||
Gitlab::CurrentSettings.current_application_settings.update_columns(gitaly_timeout_default: old_timeout)
|
||||
ApplicationSetting.expire
|
||||
end
|
||||
|
||||
def self.mute_notifications
|
||||
NotificationService.prepend(MuteNotifications)
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue