Add latest changes from gitlab-org/gitlab@master

.gitlab-ci.yml

@@ -63,6 +63,27 @@ variables:
   ELASTIC_URL: "http://elastic:changeme@elasticsearch:9200"
   DOCKER_VERSION: "19.03.0"
 
+  # Preparing custom clone path to reduce space used by all random forks
+  # on GitLab.com's Shared Runners. Our main forks - especially the security
+  # ones - will have this variable overwritten in the project settings, so that
+  # a security-related code or code using our protected variables will be never
+  # stored on the same path as the community forks.
+  # Part of the solution for the `no space left on device` problem described at
+  # https://gitlab.com/gitlab-org/gitlab/issues/197876.
+  #
+  # For this purpose the https://gitlab.com/gitlab-org-forks group was created
+  # to host a placeholder for the `/builds/gitlab-org-forks` path and ensure
+  # that no legitimate project will ever use it and - by mistake - execute its
+  # job on a shared working directory. It also requires proper configuration of
+  # the Runner that executes the job (which was prepared for our shared runners
+  # by https://ops.gitlab.net/gitlab-cookbooks/chef-repo/-/merge_requests/3977).
+  #
+  # Because of all of that PLEASE DO NOT CHANGE THE PATH.
+  #
+  # For more details and reasoning that brought this change please check
+  # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24887
+  GIT_CLONE_PATH: "/builds/gitlab-org-forks/${CI_PROJECT_NAME}"
+
 include:
   - local: .gitlab/ci/build-images.gitlab-ci.yml
   - local: .gitlab/ci/cache-repo.gitlab-ci.yml

.rubocop_todo.yml

@@ -1324,7 +1324,6 @@ Rails/SaveBang:
     - 'spec/support/shared_examples/models/members_notifications_shared_example.rb'
     - 'spec/support/shared_examples/models/mentionable_shared_examples.rb'
     - 'spec/support/shared_examples/models/project_latest_successful_build_for_shared_examples.rb'
-    - 'spec/support/shared_examples/models/relative_positioning_shared_examples.rb'
     - 'spec/support/shared_examples/models/slack_mattermost_notifications_shared_examples.rb'
     - 'spec/support/shared_examples/models/update_project_statistics_shared_examples.rb'
     - 'spec/support/shared_examples/models/with_uploads_shared_examples.rb'

Gemfile

@@ -415,7 +415,7 @@ group :test do
   gem 'webmock', '~> 3.5.1'
   gem 'rails-controller-testing'
   gem 'concurrent-ruby', '~> 1.1'
-  gem 'test-prof', '~> 0.10.0'
+  gem 'test-prof', '~> 0.12.0'
   gem 'rspec_junit_formatter'
   gem 'guard-rspec'
 

Gemfile.lock

@@ -1093,7 +1093,7 @@ GEM
     temple (0.8.2)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
-    test-prof (0.10.0)
+    test-prof (0.12.0)
     text (1.3.1)
     thin (1.7.2)
       daemons (~> 1.0, >= 1.0.9)
@@ -1430,7 +1430,7 @@ DEPENDENCIES
   stackprof (~> 0.2.15)
   state_machines-activerecord (~> 0.6.0)
   sys-filesystem (~> 1.1.6)
-  test-prof (~> 0.10.0)
+  test-prof (~> 0.12.0)
   thin (~> 1.7.0)
   timecop (~> 0.9.1)
   toml-rb (~> 1.0.0)

app/controllers/concerns/wiki_actions.rb

@@ -111,14 +111,16 @@ module WikiActions
 
   # rubocop:disable Gitlab/ModuleWithInstanceVariables
   def create
-    @page = WikiPages::CreateService.new(container: container, current_user: current_user, params: wiki_params).execute
+    response = WikiPages::CreateService.new(container: container, current_user: current_user, params: wiki_params).execute
+    @page = response.payload[:page]
 
-    if page.persisted?
+    if response.success?
       redirect_to(
         wiki_page_path(wiki, page),
         notice: _('Wiki was successfully updated.')
       )
     else
+      flash[:alert] = response.message
       render 'shared/wikis/edit'
     end
   rescue Gitlab::Git::Wiki::OperationError => e

app/models/concerns/relative_positioning.rb

@@ -3,11 +3,15 @@
 # This module makes it possible to handle items as a list, where the order of items can be easily altered
 # Requirements:
 #
-# - Only works for ActiveRecord models
-# - relative_position integer field must present on the model
-# - This module uses GROUP BY: the model should have a parent relation, example: project -> issues, project is the parent relation (issues table has a parent_id column)
+# The model must have the following named columns:
+#  - id: integer
+#  - relative_position: integer
 #
-# Setup like this in the body of your class:
+# The model must support a concept of siblings via a child->parent relationship,
+# to enable rebalancing and `GROUP BY` in queries.
+# - example: project -> issues, project is the parent relation (issues table has a parent_id column)
+#
+# Two class methods must be defined when including this concern:
 #
 #     include RelativePositioning
 #
@@ -24,66 +28,162 @@
 module RelativePositioning
   extend ActiveSupport::Concern
 
-  MIN_POSITION = 0
-  START_POSITION = Gitlab::Database::MAX_INT_VALUE / 2
+  STEPS = 10
+  IDEAL_DISTANCE = 2**(STEPS - 1) + 1
+
+  MIN_POSITION = Gitlab::Database::MIN_INT_VALUE
+  START_POSITION = 0
   MAX_POSITION = Gitlab::Database::MAX_INT_VALUE
-  IDEAL_DISTANCE = 500
+
+  MAX_GAP = IDEAL_DISTANCE * 2
+  MIN_GAP = 2
+
+  NoSpaceLeft = Class.new(StandardError)
 
   class_methods do
     def move_nulls_to_end(objects)
-      objects = objects.reject(&:relative_position)
-      return if objects.empty?
-
-      self.transaction do
-        max_relative_position = objects.first.max_relative_position
-
-        objects.each do |object|
-          relative_position = position_between(max_relative_position || START_POSITION, MAX_POSITION)
-          object.update_column(:relative_position, relative_position)
-
-          max_relative_position = relative_position
-        end
-      end
+      move_nulls(objects, at_end: true)
     end
 
     def move_nulls_to_start(objects)
-      objects = objects.reject(&:relative_position)
-      return if objects.empty?
-
-      self.transaction do
-        min_relative_position = objects.first.min_relative_position
-
-        objects.reverse_each do |object|
-          relative_position = position_between(MIN_POSITION, min_relative_position || START_POSITION)
-          object.update_column(:relative_position, relative_position)
-
-          min_relative_position = relative_position
-        end
-      end
+      move_nulls(objects, at_end: false)
     end
 
     # This method takes two integer values (positions) and
     # calculates the position between them. The range is huge as
-    # the maximum integer value is 2147483647. We are incrementing position by IDEAL_DISTANCE * 2 every time
-    # when we have enough space. If distance is less than IDEAL_DISTANCE, we are calculating an average number.
+    # the maximum integer value is 2147483647.
+    #
+    # We avoid open ranges by clamping the range to [MIN_POSITION, MAX_POSITION].
+    #
+    # Then we handle one of three cases:
+    #  - If the gap is too small, we raise NoSpaceLeft
+    #  - If the gap is larger than MAX_GAP, we place the new position at most
+    #    IDEAL_DISTANCE from the edge of the gap.
+    #  - otherwise we place the new position at the midpoint.
+    #
+    # The new position will always satisfy: pos_before <= midpoint <= pos_after
+    #
+    # As a precondition, the gap between pos_before and pos_after MUST be >= 2.
+    # If the gap is too small, NoSpaceLeft is raised.
+    #
+    # This class method should only be called by instance methods of this module, which
+    # include handling for minimum gap size.
+    #
+    # @raises NoSpaceLeft
+    # @api private
     def position_between(pos_before, pos_after)
       pos_before ||= MIN_POSITION
       pos_after ||= MAX_POSITION
 
       pos_before, pos_after = [pos_before, pos_after].sort
 
-      halfway = (pos_after + pos_before) / 2
-      distance_to_halfway = pos_after - halfway
+      gap_width = pos_after - pos_before
+      midpoint = [pos_after - 1, pos_before + (gap_width / 2)].min
 
-      if distance_to_halfway < IDEAL_DISTANCE
-        halfway
-      else
+      if gap_width < MIN_GAP
+        raise NoSpaceLeft
+      elsif gap_width > MAX_GAP
         if pos_before == MIN_POSITION
           pos_after - IDEAL_DISTANCE
         elsif pos_after == MAX_POSITION
           pos_before + IDEAL_DISTANCE
         else
-          halfway
+          midpoint
+        end
+      else
+        midpoint
+      end
+    end
+
+    private
+
+    # @api private
+    def gap_size(object, gaps:, at_end:, starting_from:)
+      total_width = IDEAL_DISTANCE * gaps
+      size = if at_end && starting_from + total_width >= MAX_POSITION
+               (MAX_POSITION - starting_from) / gaps
+             elsif !at_end && starting_from - total_width <= MIN_POSITION
+               (starting_from - MIN_POSITION) / gaps
+             else
+               IDEAL_DISTANCE
+             end
+
+      # Shift max elements leftwards if there isn't enough space
+      return [size, starting_from] if size >= MIN_GAP
+
+      order = at_end ? :desc : :asc
+      terminus = object
+        .send(:relative_siblings) # rubocop:disable GitlabSecurity/PublicSend
+        .where('relative_position IS NOT NULL')
+        .order(relative_position: order)
+        .first
+
+      if at_end
+        terminus.move_sequence_before(true)
+        max_relative_position = terminus.reset.relative_position
+        [[(MAX_POSITION - max_relative_position) / gaps, IDEAL_DISTANCE].min, max_relative_position]
+      else
+        terminus.move_sequence_after(true)
+        min_relative_position = terminus.reset.relative_position
+        [[(min_relative_position - MIN_POSITION) / gaps, IDEAL_DISTANCE].min, min_relative_position]
+      end
+    end
+
+    # @api private
+    # @param [Array<RelativePositioning>] objects The objects to give positions to. The relative
+    #                                             order will be preserved (i.e. when this method returns,
+    #                                             objects.first.relative_position < objects.last.relative_position)
+    # @param [Boolean] at_end: The placement.
+    #                          If `true`, then all objects with `null` positions are placed _after_
+    #                          all siblings with positions. If `false`, all objects with `null`
+    #                          positions are placed _before_ all siblings with positions.
+    def move_nulls(objects, at_end:)
+      objects = objects.reject(&:relative_position)
+      return if objects.empty?
+
+      representative = objects.first
+      number_of_gaps = objects.size + 1 # 1 at left, one between each, and one at right
+      position = if at_end
+                   representative.max_relative_position
+                 else
+                   representative.min_relative_position
+                 end
+
+      position ||= START_POSITION # If there are no positioned siblings, start from START_POSITION
+
+      gap, position = gap_size(representative, gaps: number_of_gaps, at_end: at_end, starting_from: position)
+
+      # Raise if we could not make enough space
+      raise NoSpaceLeft if gap < MIN_GAP
+
+      indexed = objects.each_with_index.to_a
+      starting_from = at_end ? position : position - (gap * number_of_gaps)
+
+      # Some classes are polymorphic, and not all siblings are in the same table.
+      by_model = indexed.group_by { |pair| pair.first.class }
+
+      by_model.each do |model, pairs|
+        model.transaction do
+          pairs.each_slice(100) do |batch|
+            # These are known to be integers, one from the DB, and the other
+            # calculated by us, and thus safe to interpolate
+            values = batch.map do |obj, i|
+              pos = starting_from + gap * (i + 1)
+              obj.relative_position = pos
+              "(#{obj.id}, #{pos})"
+            end.join(', ')
+
+            model.connection.exec_query(<<~SQL, "UPDATE #{model.table_name} positions")
+              WITH cte(cte_id, new_pos) AS (
+               SELECT *
+               FROM (VALUES #{values}) as t (id, pos)
+              )
+              UPDATE #{model.table_name}
+              SET relative_position = cte.new_pos
+              FROM cte
+              WHERE cte_id = id
+            SQL
+          end
         end
       end
     end
@@ -97,11 +197,12 @@ module RelativePositioning
     calculate_relative_position('MAX', &block)
   end
 
-  def prev_relative_position
+  def prev_relative_position(ignoring: nil)
     prev_pos = nil
 
     if self.relative_position
       prev_pos = max_relative_position do |relation|
+        relation = relation.id_not_in(ignoring.id) if ignoring.present?
         relation.where('relative_position < ?', self.relative_position)
       end
     end
@@ -109,11 +210,12 @@ module RelativePositioning
     prev_pos
   end
 
-  def next_relative_position
+  def next_relative_position(ignoring: nil)
     next_pos = nil
 
     if self.relative_position
       next_pos = min_relative_position do |relation|
+        relation = relation.id_not_in(ignoring.id) if ignoring.present?
         relation.where('relative_position > ?', self.relative_position)
       end
     end
@@ -125,24 +227,44 @@ module RelativePositioning
     return move_after(before) unless after
     return move_before(after) unless before
 
-    # If there is no place to insert an item we need to create one by moving the item
-    # before this and all preceding items until there is a gap
     before, after = after, before if after.relative_position < before.relative_position
-    if (after.relative_position - before.relative_position) < 2
-      after.move_sequence_before
-      before.reset
+
+    pos_left = before.relative_position
+    pos_right = after.relative_position
+
+    if pos_right - pos_left < MIN_GAP
+      # Not enough room! Make space by shifting all previous elements to the left
+      # if there is enough space, else to the right
+      gap = after.send(:find_next_gap_before) # rubocop:disable GitlabSecurity/PublicSend
+
+      if gap.present?
+        after.move_sequence_before(next_gap: gap)
+        pos_left -= optimum_delta_for_gap(gap)
+      else
+        before.move_sequence_after
+        pos_right = after.reset.relative_position
+      end
     end
 
-    self.relative_position = self.class.position_between(before.relative_position, after.relative_position)
+    new_position = self.class.position_between(pos_left, pos_right)
+
+    self.relative_position = new_position
   end
 
   def move_after(before = self)
     pos_before = before.relative_position
-    pos_after = before.next_relative_position
+    pos_after = before.next_relative_position(ignoring: self)
 
-    if pos_after && (pos_after - pos_before) < 2
-      before.move_sequence_after
-      pos_after = before.next_relative_position
+    if pos_before == MAX_POSITION || gap_too_small?(pos_after, pos_before)
+      gap = before.send(:find_next_gap_after) # rubocop:disable GitlabSecurity/PublicSend
+
+      if gap.nil?
+        before.move_sequence_before(true)
+        pos_before = before.reset.relative_position
+      else
+        before.move_sequence_after(next_gap: gap)
+        pos_after += optimum_delta_for_gap(gap)
+      end
     end
 
     self.relative_position = self.class.position_between(pos_before, pos_after)
@@ -150,80 +272,168 @@ module RelativePositioning
 
   def move_before(after = self)
     pos_after = after.relative_position
-    pos_before = after.prev_relative_position
+    pos_before = after.prev_relative_position(ignoring: self)
 
-    if pos_before && (pos_after - pos_before) < 2
-      after.move_sequence_before
-      pos_before = after.prev_relative_position
+    if pos_after == MIN_POSITION || gap_too_small?(pos_before, pos_after)
+      gap = after.send(:find_next_gap_before) # rubocop:disable GitlabSecurity/PublicSend
+
+      if gap.nil?
+        after.move_sequence_after(true)
+        pos_after = after.reset.relative_position
+      else
+        after.move_sequence_before(next_gap: gap)
+        pos_before -= optimum_delta_for_gap(gap)
+      end
     end
 
     self.relative_position = self.class.position_between(pos_before, pos_after)
   end
 
   def move_to_end
-    self.relative_position = self.class.position_between(max_relative_position || START_POSITION, MAX_POSITION)
+    max_pos = max_relative_position
+
+    self.relative_position = max_pos.nil? ? START_POSITION : self.class.position_between(max_pos, MAX_POSITION)
   end
 
   def move_to_start
-    self.relative_position = self.class.position_between(min_relative_position || START_POSITION, MIN_POSITION)
+    min_pos = max_relative_position
+
+    self.relative_position = min_pos.nil? ? START_POSITION : self.class.position_between(MIN_POSITION, min_pos)
   end
 
   # Moves the sequence before the current item to the middle of the next gap
-  # For example, we have 5 11 12 13 14 15 and the current item is 15
-  # This moves the sequence 11 12 13 14 to 8 9 10 11
-  def move_sequence_before
-    next_gap = find_next_gap_before
+  # For example, we have
+  #
+  #   5 . . . . . 11 12 13 14 [15] 16 . 17
+  #               -----------
+  #
+  # This moves the sequence [11 12 13 14] to [8 9 10 11], so we have:
+  #
+  #   5 . . 8 9 10 11 . . . [15] 16 . 17
+  #         ---------
+  #
+  # Creating a gap to the left of the current item. We can understand this as
+  # dividing the 5 spaces between 5 and 11 into two smaller gaps of 2 and 3.
+  #
+  # If `include_self` is true, the current item will also be moved, creating a
+  # gap to the right of the current item:
+  #
+  #   5 . . 8 9 10 11 [14] . . . 16 . 17
+  #         --------------
+  #
+  # As an optimization, the gap can be precalculated and passed to this method.
+  #
+  # @api private
+  # @raises NoSpaceLeft if the sequence cannot be moved
+  def move_sequence_before(include_self = false, next_gap: find_next_gap_before)
+    raise NoSpaceLeft unless next_gap.present?
+
     delta = optimum_delta_for_gap(next_gap)
 
-    move_sequence(next_gap[:start], relative_position, -delta)
+    move_sequence(next_gap[:start], relative_position, -delta, include_self)
   end
 
   # Moves the sequence after the current item to the middle of the next gap
-  # For example, we have 11 12 13 14 15 21 and the current item is 11
-  # This moves the sequence 12 13 14 15 to 15 16 17 18
-  def move_sequence_after
-    next_gap = find_next_gap_after
+  # For example, we have:
+  #
+  #   8 . 10 [11] 12 13 14 15 . . . . . 21
+  #               -----------
+  #
+  # This moves the sequence [12 13 14 15] to [15 16 17 18], so we have:
+  #
+  #   8 . 10 [11] . . . 15 16 17 18 . . 21
+  #                     -----------
+  #
+  # Creating a gap to the right of the current item. We can understand this as
+  # dividing the 5 spaces between 15 and 21 into two smaller gaps of 3 and 2.
+  #
+  # If `include_self` is true, the current item will also be moved, creating a
+  # gap to the left of the current item:
+  #
+  #   8 . 10 . . . [14] 15 16 17 18 . . 21
+  #                ----------------
+  #
+  # As an optimization, the gap can be precalculated and passed to this method.
+  #
+  # @api private
+  # @raises NoSpaceLeft if the sequence cannot be moved
+  def move_sequence_after(include_self = false, next_gap: find_next_gap_after)
+    raise NoSpaceLeft unless next_gap.present?
+
     delta = optimum_delta_for_gap(next_gap)
 
-    move_sequence(relative_position, next_gap[:start], delta)
+    move_sequence(relative_position, next_gap[:start], delta, include_self)
   end
 
   private
 
-  # Supposing that we have a sequence of items: 1 5 11 12 13 and the current item is 13
-  # This would return: `{ start: 11, end: 5 }`
+  def gap_too_small?(pos_a, pos_b)
+    return false unless pos_a && pos_b
+
+    (pos_a - pos_b).abs < MIN_GAP
+  end
+
+  # Find the first suitable gap to the left of the current position.
+  #
+  # Satisfies the relations:
+  #  - gap[:start] <= relative_position
+  #  - abs(gap[:start] - gap[:end]) >= MIN_GAP
+  #  - MIN_POSITION <= gap[:start] <= MAX_POSITION
+  #  - MIN_POSITION <= gap[:end] <= MAX_POSITION
+  #
+  # Supposing that the current item is 13, and we have a sequence of items:
+  #
+  #   1 . . . 5 . . . . 11 12 [13] 14 . . 17
+  #           ^---------^
+  #
+  # Then we return: `{ start: 11, end: 5 }`
+  #
+  # Here start refers to the end of the gap closest to the current item.
   def find_next_gap_before
     items_with_next_pos = scoped_items
                             .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos')
                             .where('relative_position <= ?', relative_position)
                             .order(relative_position: :desc)
 
-    find_next_gap(items_with_next_pos).tap do |gap|
-      gap[:end] ||= MIN_POSITION
-    end
+    find_next_gap(items_with_next_pos, MIN_POSITION)
   end
 
-  # Supposing that we have a sequence of items: 13 14 15 20 24 and the current item is 13
-  # This would return: `{ start: 15, end: 20 }`
+  # Find the first suitable gap to the right of the current position.
+  #
+  # Satisfies the relations:
+  #  - gap[:start] >= relative_position
+  #  - abs(gap[:start] - gap[:end]) >= MIN_GAP
+  #  - MIN_POSITION <= gap[:start] <= MAX_POSITION
+  #  - MIN_POSITION <= gap[:end] <= MAX_POSITION
+  #
+  # Supposing the current item is 13, and that we have a sequence of items:
+  #
+  #   9 . . . [13] 14 15 . . . . 20 . . . 24
+  #                    ^---------^
+  #
+  # Then we return: `{ start: 15, end: 20 }`
+  #
+  # Here start refers to the end of the gap closest to the current item.
   def find_next_gap_after
     items_with_next_pos = scoped_items
                             .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos')
                             .where('relative_position >= ?', relative_position)
                             .order(:relative_position)
 
-    find_next_gap(items_with_next_pos).tap do |gap|
-      gap[:end] ||= MAX_POSITION
-    end
+    find_next_gap(items_with_next_pos, MAX_POSITION)
   end
 
-  def find_next_gap(items_with_next_pos)
-    gap = self.class.from(items_with_next_pos, :items_with_next_pos)
-                    .where('ABS(pos - next_pos) > 1 OR next_pos IS NULL')
-                    .limit(1)
-                    .pluck(:pos, :next_pos)
-                    .first
+  def find_next_gap(items_with_next_pos, end_is_nil)
+    gap = self.class
+      .from(items_with_next_pos, :items)
+      .where('next_pos IS NULL OR ABS(pos::bigint - next_pos::bigint) >= ?', MIN_GAP)
+      .limit(1)
+      .pluck(:pos, :next_pos)
+      .first
 
-    { start: gap[0], end: gap[1] }
+    return if gap.nil? || gap.first == end_is_nil
+
+    { start: gap.first, end: gap.second || end_is_nil }
   end
 
   def optimum_delta_for_gap(gap)
@@ -232,9 +442,10 @@ module RelativePositioning
     [delta, IDEAL_DISTANCE].min
   end
 
-  def move_sequence(start_pos, end_pos, delta)
-    scoped_items
-      .where.not(id: self.id)
+  def move_sequence(start_pos, end_pos, delta, include_self = false)
+    relation = include_self ? scoped_items : relative_siblings
+
+    relation
       .where('relative_position BETWEEN ? AND ?', start_pos, end_pos)
       .update_all("relative_position = relative_position + #{delta}")
   end
@@ -255,6 +466,10 @@ module RelativePositioning
       .first&.last
   end
 
+  def relative_siblings(relation = scoped_items)
+    relation.id_not_in(id)
+  end
+
   def scoped_items
     self.class.relative_positioning_query_base(self)
   end

app/services/wiki_pages/create_service.rb

@@ -10,7 +10,11 @@ module WikiPages
         execute_hooks(page)
       end
 
-      page
+      if page.persisted?
+        ServiceResponse.success(payload: { page: page })
+      else
+        ServiceResponse.error(message: _('Could not create wiki page'), payload: { page: page })
+      end
     end
 
     def usage_counter_action

app/views/profiles/preferences/_sourcegraph.html.haml

@@ -4,7 +4,7 @@
 .col-sm-12
   %hr
 
-.col-lg-4.profile-settings-sidebar
+.col-lg-4.profile-settings-sidebar#integrations
   %h4.gl-mt-0
     = s_('Preferences|Integrations')
   %p

app/views/profiles/preferences/show.html.haml

@@ -2,7 +2,7 @@
 - @content_class = "limit-container-width" unless fluid_layout
 
 = form_for @user, url: profile_preferences_path, remote: true, method: :put, html: { class: 'row gl-mt-3 js-preferences-form' } do |f|
-  .col-lg-4.application-theme
+  .col-lg-4.application-theme#navigation-theme
     %h4.gl-mt-0
       = s_('Preferences|Navigation theme')
     %p
@@ -18,7 +18,7 @@
   .col-sm-12
     %hr
 
-  .col-lg-4.profile-settings-sidebar
+  .col-lg-4.profile-settings-sidebar#syntax-highlighting-theme
     %h4.gl-mt-0
       = s_('Preferences|Syntax highlighting theme')
     %p
@@ -92,7 +92,7 @@
   .col-sm-12
     %hr
 
-  .col-lg-4.profile-settings-sidebar
+  .col-lg-4.profile-settings-sidebar#localization
     %h4.gl-mt-0
       = _('Localization')
     %p

changelogs/unreleased/212388-docs-and-follow-up-spdx.yml

@@ -0,0 +1,5 @@
+---
+title: Enable read SPDX catalogue from local copy
+merge_request: 39463
+author:
+type: added

changelogs/unreleased/235858-add-anchors-to-profile-preferences.yml

@@ -0,0 +1,5 @@
+---
+title: Add anchors to profile preferences
+merge_request: 39589
+author:
+type: changed

changelogs/unreleased/ajk-relative-positioning-improvements.yml

@@ -0,0 +1,5 @@
+---
+title: Performance and robustness improvements for relative positioning
+merge_request: 37724
+author:
+type: performance

config/dependency_decisions.yml

@@ -102,20 +102,6 @@
     :why: The OSL license is a copyleft license
     :versions: []
     :when: 2016-10-28 11:02:15.540105000 Z
-- - :license
-  - raphael-rails
-  - MIT
-  - :who: Connor Shea
-    :why: https://github.com/mockdeep/raphael-rails/blob/master/license.txt
-    :versions: []
-    :when: 2016-04-17 21:30:07.575392000 Z
-- - :license
-  - rouge
-  - MIT
-  - :who: Connor Shea
-    :why: https://github.com/jneen/rouge/blob/master/LICENSE
-    :versions: []
-    :when: 2016-04-17 21:31:29.490394000 Z
 - - :license
   - pyu-ruby-sasl
   - MIT
@@ -123,20 +109,6 @@
     :why: https://github.com/pyu10055/ruby-sasl/blob/master/MIT-LICENSE
     :versions: []
     :when: 2016-04-17 21:41:55.266420000 Z
-- - :license
-  - six
-  - MIT
-  - :who: Connor Shea
-    :why: https://github.com/randx/six/blob/master/LICENSE
-    :versions: []
-    :when: 2016-04-17 21:42:31.420186000 Z
-- - :license
-  - rdoc
-  - ruby
-  - :who: Connor Shea
-    :why: https://github.com/rdoc/rdoc/blob/master/LICENSE.rdoc
-    :versions: []
-    :when: 2016-04-17 21:43:30.480413000 Z
 - - :license
   - expression_parser
   - MIT
@@ -151,13 +123,6 @@
     :why: https://github.com/minad/creole#license
     :versions: []
     :when: 2016-04-17 21:49:10.329759000 Z
-- - :license
-  - eventmachine
-  - ruby
-  - :who: Connor Shea
-    :why: https://github.com/eventmachine/eventmachine/blob/master/LICENSE
-    :versions: []
-    :when: 2016-04-17 21:49:10.329759001 Z
 - - :license
   - unicorn
   - ruby
@@ -172,13 +137,6 @@
     :why: https://github.com/kzk/unicorn-worker-killer/blob/master/LICENSE
     :versions: []
     :when: 2016-05-02 05:45:38.323867000 Z
-- - :license
-  - json
-  - ruby
-  - :who: Connor Shea
-    :why: https://github.com/flori/json/tree/master#license
-    :versions: []
-    :when: 2016-05-02 05:50:07.826564000 Z
 - - :license
   - unf
   - BSD
@@ -193,48 +151,6 @@
     :why: https://github.com/jmcnevin/rubypants/blob/master/LICENSE.rdoc
     :versions: []
     :when: 2016-05-02 05:56:50.696858000 Z
-- - :approve
-  - after
-  - :who: Matt Lee
-    :why: https://github.com/Raynos/after/blob/master/LICENCE
-    :versions: []
-    :when: 2017-01-14 20:00:32.473125000 Z
-- - :approve
-  - amdefine
-  - :who: Matt Lee
-    :why: MIT License
-    :versions: []
-    :when: 2017-01-14 20:08:31.810633000 Z
-- - :approve
-  - base64id
-  - :who: Matt Lee
-    :why: https://github.com/faeldt/base64id/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:08:33.174760000 Z
-- - :approve
-  - blob
-  - :who: Matt Lee
-    :why: https://github.com/webmodules/blob/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:08:34.564048000 Z
-- - :approve
-  - callsite
-  - :who: Matt Lee
-    :why: https://github.com/tj/callsite/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:08:35.976025000 Z
-- - :approve
-  - component-bind
-  - :who: Matt Lee
-    :why: https://github.com/component/bind/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:08:37.291219000 Z
-- - :approve
-  - component-inherit
-  - :who: Matt Lee
-    :why: https://github.com/component/inherit/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:10:41.804804000 Z
 - - :license
   - fsevents
   - MIT
@@ -242,85 +158,12 @@
     :why: https://github.com/strongloop/fsevents/blob/master/LICENSE
     :versions: []
     :when: 2017-01-14 20:50:20.037775000 Z
-- - :approve
-  - indexof
-  - :who: Matt Lee
-    :why: https://github.com/component/indexof/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:10:43.209900000 Z
-- - :approve
-  - is-integer
-  - :who: Matt Lee
-    :why: https://github.com/parshap/js-is-integer/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:10:44.540916000 Z
-- - :approve
-  - jsonify
-  - :who: Matt Lee
-    :why: Public Domain - no formal license on this one. probably okay as its been
-      the same for along time. would prefer to see CC0
-    :versions: []
-    :when: 2017-01-14 20:10:45.857261000 Z
-- - :approve
-  - object-component
-  - :who: Matt Lee
-    :why: https://github.com/component/object/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:10:47.190148000 Z
-- - :approve
-  - optimist
-  - :who: Matt Lee
-    :why: https://github.com/substack/node-optimist/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:10:48.563077000 Z
-- - :approve
-  - path-is-inside
-  - :who: Matt Lee
-    :why: https://github.com/domenic/path-is-inside/blob/master/LICENSE.txt
-    :versions: []
-    :when: 2017-01-14 20:10:49.910497000 Z
-- - :approve
-  - rc
-  - :who: Matt Lee
-    :why: https://github.com/dominictarr/rc/blob/master/LICENSE.MIT
-    :versions: []
-    :when: 2017-01-14 20:10:51.244695000 Z
-- - :approve
-  - ripemd160
-  - :who: Matt Lee
-    :why: https://github.com/crypto-browserify/ripemd160/blob/master/LICENSE.md
-    :versions: []
-    :when: 2017-01-14 20:10:52.560282000 Z
 - - :approve
   - select2
   - :who: Matt Lee
     :why: https://github.com/select2/select2/blob/master/LICENSE.md
     :versions: []
     :when: 2017-01-14 20:10:53.909618000 Z
-- - :approve
-  - tweetnacl
-  - :who: Matt Lee
-    :why: https://github.com/dchest/tweetnacl-js/blob/master/LICENSE
-    :versions: []
-    :when: 2017-01-14 20:10:57.812077000 Z
-- - :approve
-  - wordwrap
-  - :who: Mike Greiling
-    :why: https://github.com/substack/node-wordwrap/blob/0.0.3/LICENSE
-    :versions: []
-    :when: 2017-02-08 20:17:13.084968000 Z
-- - :approve
-  - spdx-expression-parse
-  - :who: Mike Greiling
-    :why: https://github.com/kemitchell/spdx-expression-parse.js/blob/v1.0.4/LICENSE
-    :versions: []
-    :when: 2017-02-08 22:33:01.806977000 Z
-- - :approve
-  - spdx-license-ids
-  - :who: Mike Greiling
-    :why: https://github.com/shinnn/spdx-license-ids/blob/v1.2.2/LICENSE
-    :versions: []
-    :when: 2017-02-08 22:35:00.225232000 Z
 - - :approve
   - opener
   - :who: Mike Greiling
@@ -345,67 +188,6 @@
     :why: https://github.com/nodeca/pako/blob/master/LICENSE
     :versions: []
     :when: 2017-04-05 10:43:45.897720000 Z
-- - :approve
-  - caniuse-db
-  - :who: Mike Greiling
-    :why: https://github.com/Fyrd/caniuse/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-07 16:05:14.185549000 Z
-- - :approve
-  - domelementtype
-  - :who: Mike Greiling
-    :why: https://github.com/fb55/domelementtype/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-07 16:19:17.992640000 Z
-- - :approve
-  - domhandler
-  - :who: Mike Greiling
-    :why: https://github.com/fb55/domhandler/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-07 16:19:19.628953000 Z
-- - :approve
-  - domutils
-  - :who: Mike Greiling
-    :why: https://github.com/fb55/domutils/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-07 16:19:21.159356000 Z
-- - :approve
-  - entities
-  - :who: Mike Greiling
-    :why: https://github.com/fb55/entities/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-07 16:19:23.900571000 Z
-- - :approve
-  - ansi-html
-  - :who: Mike Greiling
-    :why: https://github.com/Tjatse/ansi-html/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-10 05:42:12.898178000 Z
-- - :license
-  - map-stream
-  - MIT
-  - :who: Mike Greiling
-    :why: https://github.com/dominictarr/map-stream/blob/master/LICENCE
-    :versions: []
-    :when: 2017-04-10 06:27:52.269085000 Z
-- - :approve
-  - pause-stream
-  - :who: Mike Greiling
-    :why: https://github.com/dominictarr/pause-stream/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-10 06:28:39.825894000 Z
-- - :approve
-  - undefsafe
-  - :who: Mike Greiling
-    :why: https://github.com/remy/undefsafe/blob/master/LICENSE
-    :versions: []
-    :when: 2017-04-10 06:30:00.002555000 Z
-- - :approve
-  - thunky
-  - :who: Mike Greiling
-    :why: https://github.com/mafintosh/thunky/blob/master/README.md#license
-    :versions: []
-    :when: 2017-08-07 05:56:09.907045000 Z
 - - :whitelist
   - Unlicense
   - :who: Nick Thomas <nick@gitlab.com>
@@ -418,49 +200,6 @@
     :why: https://gitlab.com/gitlab-com/organization/issues/117
     :versions: []
     :when: 2017-09-04 12:59:51.150798717 Z
-- - :approve
-  - console-browserify
-  - :who: Mike Greiling
-    :why: https://github.com/Raynos/console-browserify/blob/f0a8898487e2a47b8a5dc8734b91059fa2825506/LICENCE
-    :versions: []
-    :when: 2017-09-16 05:13:07.073651000 Z
-- - :approve
-  - duplexer
-  - :who: Mike Greiling
-    :why: https://github.com/Raynos/duplexer/blob/master/LICENCE
-    :versions: []
-    :when: 2017-09-16 05:14:15.774643000 Z
-- - :approve
-  - json3
-  - :who: Mike Greiling
-    :why: https://github.com/bestiejs/json3/blob/v3.3.2/LICENSE
-    :versions: []
-    :when: 2017-09-16 05:15:16.273892000 Z
-- - :approve
-  - mime
-  - :who: Mike Greiling
-    :why: https://github.com/broofa/node-mime/blob/v1.3.4/LICENSE
-    :versions: []
-    :when: 2017-09-16 05:16:21.135542000 Z
-- - :approve
-  - querystring-es3
-  - :who: Mike Greiling
-    :why: https://github.com/mike-spainhower/querystring/blob/v0.2.0/License.md
-    :versions: []
-    :when: 2017-09-16 05:17:20.372089000 Z
-- - :approve
-  - utils-merge
-  - :who: Mike Greiling
-    :why: https://github.com/jaredhanson/utils-merge/blob/v1.0.0/LICENSE
-    :versions: []
-    :when: 2017-09-16 05:18:26.193764000 Z
-- - :license
-  - "@gitlab/svgs"
-  - MIT
-  - :who: Tim Zallmann
-    :why: Our own library - GitLab License https://gitlab.com/gitlab-org/gitlab-svgs
-    :versions: []
-    :when: 2017-09-19 14:36:32.795496000 Z
 - - :license
   - pikaday
   - MIT
@@ -468,51 +207,6 @@
     :why: MIT License
     :versions: []
     :when: 2017-10-17 17:46:12.367554000 Z
-- - :license
-  - component-emitter
-  - MIT
-  - :who: Winnie Hellmann
-    :why: package.json does not specify the license (README.md does)
-    :versions:
-    - 1.1.2
-    :when: 2017-11-13 12:23:10.502463000 Z
-- - :license
-  - json-schema
-  - BSD
-  - :who: Winnie Hellmann
-    :why: https://github.com/kriszyp/json-schema/blob/v0.2.3/package.json#L18-L19
-    :versions:
-    - 0.2.3
-    :when: 2017-11-16 12:52:18.286091000 Z
-- - :license
-  - node-forge
-  - New BSD
-  - :who: Winnie Hellmann
-    :why: https://github.com/digitalbazaar/forge/blob/0.6.33/LICENSE
-    :versions:
-    - 0.6.33
-    :when: 2017-11-16 12:56:17.974767000 Z
-- - :license
-  - sntp
-  - BSD
-  - :who: Winnie Hellmann
-    :why: https://github.com/hueniverse/sntp/blob/v1.0.9/package.json#L28-L29
-    :versions:
-    - 1.0.9
-    :when: 2017-11-16 13:02:06.765282000 Z
-- - :license
-  - JSONStream
-  - MIT
-  - :who: Tim Zallmann
-    :why: https://github.com/dominictarr/JSONStream/blob/master/LICENSE.MIT
-    :versions: []
-    :when: 2018-01-17 22:46:12.367554000 Z
-- - :approve
-  - uws
-  - :who: Tim Zallmann
-    :why: zlib license + Development Lib + https://github.com/uNetworking/uWebSockets/blob/master/LICENSE
-    :versions: []
-    :when: 2018-01-17 23:46:12.367554000 Z
 - - :approve
   - atob
   - :who: Mike Greiling
@@ -525,19 +219,6 @@
     :why: https://github.com/mafintosh/cyclist/blob/master/LICENSE
     :versions: []
     :when: 2018-02-20 21:37:43.774978000 Z
-- - :license
-  - bitsyntax
-  - MIT
-  - :who: Mike Greiling
-    :why: https://github.com/squaremo/bitsyntax-js/blob/master/LICENSE-MIT
-    :versions: []
-    :when: 2018-02-20 22:20:25.958123000 Z
-- - :approve
-  - "@webassemblyjs/ieee754"
-  - :who: Mike Greiling
-    :why: https://github.com/xtuc/webassemblyjs/blob/master/LICENSE
-    :versions: []
-    :when: 2018-06-08 05:30:56.764116000 Z
 - - :approve
   - lz-string
   - :who: Phil Hughes
@@ -579,20 +260,6 @@
       in compiled/distributed product so attribution not needed.
     :versions: []
     :when: 2018-10-02 19:23:11.221660000 Z
-- - :approve
-  - node-releases
-  - :who: Mike Greiling
-    :why: CC-BY-4.0 license. Tool only used during build process, code is not present
-      in compiled/distributed product so attribution not needed.
-    :versions: []
-    :when: 2018-10-02 19:23:54.840151000 Z
-- - :license
-  - echarts
-  - Apache 2.0
-  - :who: Adriel Santiago
-    :why: https://github.com/apache/incubator-echarts/blob/master/LICENSE
-    :versions: []
-    :when: 2018-12-07 20:46:12.421256000 Z
 - - :license
   - contracts
   - BSD

doc/administration/lfs/index.md

@@ -157,6 +157,23 @@ On Omnibus installations, the settings are prefixed by `lfs_object_store_`:
    This will migrate existing LFS objects to object storage. New LFS objects
    will be forwarded to object storage unless
    `gitlab_rails['lfs_object_store_background_upload']` and `gitlab_rails['lfs_object_store_direct_upload']` is set to `false`.
+1. Optional: Verify all files migrated properly.
+   From [PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database)
+   (`sudo gitlab-psql -d gitlabhq_production`) verify `objectstg` below (where `file_store=2`) has count of all artifacts:
+
+   ```shell
+   gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM lfs_objects;
+
+   total | filesystem | objectstg
+   ------+------------+-----------
+    2409 |          0 |      2409
+   ```
+
+   Verify no files on disk in `artifacts` folder:
+
+   ```shell
+   sudo find /var/opt/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp/cache | wc -l
+   ```
 
 ### S3 for installations from source
 
@@ -193,6 +210,22 @@ For source installations the settings are nested under `lfs:` and then
    This will migrate existing LFS objects to object storage. New LFS objects
    will be forwarded to object storage unless `background_upload` and `direct_upload` is set to
    `false`.
+1. Optional: Verify all files migrated properly.
+   From PostgreSQL console (`sudo -u git -H psql -d gitlabhq_production`) verify `objectstg` below (where `file_store=2`) has count of all artifacts:
+
+   ```shell
+   gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM lfs_objects;
+
+   total | filesystem | objectstg
+   ------+------------+-----------
+    2409 |          0 |      2409
+   ```
+
+   Verify no files on disk in `artifacts` folder:
+
+   ```shell
+   sudo find /var/opt/gitlab/gitlab-rails/shared/lfs-objects -type f | grep -v tmp/cache | wc -l
+   ```
 
 ### Migrating back to local storage
 

doc/administration/operations/filesystem_benchmarking.md

@@ -26,7 +26,7 @@ To install:
 Then run the following:
 
 ```shell
-fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test --filename=/path/to/git-data/testfile --bs=4k --iodepth=64 --size=4G --readwrite=randrw --rwmixread=75
+fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test --bs=4k --iodepth=64 --readwrite=randrw --rwmixread=75 --size=4G --filename=/path/to/git-data/testfile
 ```
 
 This will create a 4GB file in `/path/to/git-data/testfile`. It performs

doc/install/installation.md

@@ -462,7 +462,7 @@ Clone Enterprise Edition:
 
 ```shell
 # Clone GitLab repository
-sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-ee.git -b X-Y-stable gitlab
+sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab.git -b X-Y-stable gitlab
 ```
 
 Make sure to replace `X-Y-stable` with the stable branch that matches the
@@ -522,9 +522,6 @@ sudo -u git -H cp config/puma.rb.example config/puma.rb
 # cores you have available. You can get that number via the `nproc` command.
 sudo -u git -H editor config/puma.rb
 
-# Copy the example Rack attack config
-sudo -u git -H cp config/initializers/rack_attack.rb.example config/initializers/rack_attack.rb
-
 # Configure Git global settings for git user
 # 'autocrlf' is needed for the web editor
 sudo -u git -H git config --global core.autocrlf input

doc/integration/elasticsearch.md

@@ -809,11 +809,11 @@ There is a [more structured, lower-level troubleshooting document](../administra
 
 ### Known Issues
 
-- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/gitlab-org/gitlab/-/issues/10693)**
+- **[Elasticsearch `code_analyzer` doesn't account for all code cases](https://gitlab.com/groups/gitlab-org/-/epics/3621)**
 
-   The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have noticed [several edge cases](https://gitlab.com/gitlab-org/gitlab/-/issues/10693#note_158382332) that are not returning expected search results due to our pattern and filter configuration.
+   The `code_analyzer` pattern and filter configuration is being evaluated for improvement. We have fixed [most edge cases](https://gitlab.com/groups/gitlab-org/-/epics/3621#note_363429094) that were not returning expected search results due to our pattern and filter configuration.
 
-   An improved strategy for the `code_analyzer` pattern and filters are being discussed in [issue 29443](https://gitlab.com/gitlab-org/gitlab/-/issues/29443).
+   Improvements to the `code_analyzer` pattern and filters is being discussed in [epic 3621](https://gitlab.com/groups/gitlab-org/-/epics/3621).
 
 ### Reverting to basic search
 

doc/raketasks/README.md

@@ -35,6 +35,7 @@ The following are available Rake tasks:
 | [Praefect Rake tasks](../administration/raketasks/praefect.md)                                      | [Praefect](../administration/gitaly/praefect.md)-related tasks.                           |
 | [Project import/export](../administration/raketasks/project_import_export.md)                       | Prepare for [project exports and imports](../user/project/settings/import_export.md).     |
 | [Sample Prometheus data](generate_sample_prometheus_data.md)                                        | Generate sample Prometheus data.                                                          |
+| [SPDX license list import](spdx.md) **(PREMIUM ONLY)**                                              | Import a local copy of the [SPDX license list](https://spdx.org/licenses/) for matching [License Compliance policies](../user/compliance/license_compliance/index.md).|                                                     |
 | [Repository storage](../administration/raketasks/storage.md)                                        | List and migrate existing projects and attachments from legacy storage to hashed storage. |
 | [Uploads migrate](../administration/raketasks/uploads/migrate.md)                                   | Migrate uploads between storage local and object storage.                                 |
 | [Uploads sanitize](../administration/raketasks/uploads/sanitize.md)                                 | Remove EXIF data from images uploaded to earlier versions of GitLab.                      |

doc/raketasks/spdx.md

@@ -0,0 +1,18 @@
+# SPDX license list import **(PREMIUM ONLY)**
+
+GitLab provides a Rake task for uploading a fresh copy of the [SPDX license list](https://spdx.org/licenses/)
+to a GitLab instance. This list is needed for matching the names of [License Compliance policies](../user/compliance/license_compliance/index.md).
+
+To import a fresh copy of the PDX license list, run:
+
+```shell
+# omnibus-gitlab
+sudo gitlab-rake gitlab:spdx:import
+
+# source installations
+bundle exec rake gitlab:spdx:import RAILS_ENV=production
+```
+
+To perform this task in the [offline environment](../user/application_security/offline_deployments/#defining-offline-environments),
+an outbound connection to [`licenses.json`](https://spdx.org/licenses/licenses.json) should be
+allowed.

doc/user/application_security/dast/index.md

@@ -450,7 +450,7 @@ DAST can be [configured](#customizing-the-dast-settings) using environment varia
 | `DAST_PASSWORD` | string | The password to authenticate to in the website. |
 | `DAST_USERNAME_FIELD` | string | The name of username field at the sign-in HTML form. |
 | `DAST_PASSWORD_FIELD` | string | The name of password field at the sign-in HTML form. |
-| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (introduced in GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). |
+| `DAST_MASK_HTTP_HEADERS` | string | Comma-separated list of request and response headers to be masked (GitLab 13.1). Must contain **all** headers to be masked. Refer to [list of headers that are masked by default](#hide-sensitive-information). |
 | `DAST_AUTH_EXCLUDE_URLS` | URLs | The URLs to skip during the authenticated scan; comma-separated, no spaces in between. Not supported for API scans. |
 | `DAST_FULL_SCAN_ENABLED` | boolean | Set to `true` to run a [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of a [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Default: `false` |
 | `DAST_FULL_SCAN_DOMAIN_VALIDATION_REQUIRED` | boolean | Set to `true` to require [domain validation](#domain-validation) when running DAST full scans. Not supported for API scans. Default: `false` |
@@ -603,27 +603,76 @@ Alternatively, you can use the variable `SECURE_ANALYZERS_PREFIX` to override th
 ## On-Demand Scans
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218465) in GitLab 13.2.
-> - It's deployed behind a feature flag, disabled by default.
-> - It's disabled on GitLab.com.
+> - [Improved](https://gitlab.com/gitlab-org/gitlab/-/issues/218465) in GitLab 13.3.
+> - It's deployed behind a feature flag, enabled by default.
+> - It's enabled on GitLab.com.
 > - It's able to be enabled or disabled per-project.
 > - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-on-demand-scans).
 
-Passive DAST scans may be run on demand against a target website, outside the DevOps lifecycle. These scans are
-always associated with the default or `master` branch of your project and the results can be seen in the project dashboard.
+You can run a passive DAST scan against a target website, outside the DevOps lifecycle. These scans
+are always associated with the default branch of your project and the results are available in the
+project dashboard.
+
+### Site profile
+
+An on-demand scan requires a site profile, which includes a profile name and target URL. The profile
+name allows you to describe the site to be scanned. The target URL specifies the URL against which
+the DAST scan is run.
+
+### Run an on-demand scan
 
 NOTE: **Note:**
-You cannot run an on-demand DAST scan against a protected branch unless you have permission to do so. The `master` branch is protected by default. For more details, see [Pipeline security on protected branches](../../../ci/pipelines/index.md#pipeline-security-on-protected-branches).
+You must have permission to run an on-demand DAST scan against a protected branch.
+The default branch is automatically protected. For more details, see [Pipeline security on protected branches](../../../ci/pipelines/index.md#pipeline-security-on-protected-branches).
 
-![DAST On-Demand Scan](img/dast_on_demand_v13_2.png)
+Running an on-demand scan requires an existing site profile. If a site profile for the target URL
+doesn't exist, first [create a site profile](#create-a-site-profile). An on-demand DAST scan has
+a fixed timeout of 60 seconds.
 
-### Enable or disable On-Demand Scans
+- Navigate to your project's home page, then click **On-demand Scans** in the left sidebar.
+- Click **Create new DAST scan**.
+- Select a site profile from the profiles dropdown.
+- Click **Run scan**.
+
+#### Create a site profile
+
+- Navigate to your project's home page, then click **On-demand Scans** in the left sidebar.
+- Click **Create new DAST scan**.
+- Click **New Site Profile**.
+- Type in a unique **Profile name** and **Target URL** then click **Save profile**.
+
+#### Delete a site profile
+
+- Navigate to your project's home page, then click **On-demand Scans** in the left sidebar.
+- Click **Create new DAST scan**.
+- Click **Delete** in the matching site profile's row.
+
+### Enable or disable On-demand Scans and site profiles
+
+On-demand Scans with site profiles is enabled by default. You can disable On-demand Scans
+instance-wide, or disable it for specific projects if you prefer. DAST site profiles are not
+available if the On-demand Scans feature is disabled.
+
+Use of On-demand Scans with site profiles requires **both** the following feature flags enabled:
+
+- security_on_demand_scans_feature_flag
+- security_on_demand_scans_site_profiles_feature_flag
 
-On-Demand Scans is under development and not ready for production use. It is
-deployed behind a feature flag that is **disabled by default**.
 [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
-can enable it for your instance. On-Demand Scans can be enabled or disabled per-project
+can disable or enable the feature flags.
 
-To enable it:
+#### Enable or disable On-demand Scans
+
+To disable On-demand Scans:
+
+```ruby
+# Instance-wide
+Feature.disable(:security_on_demand_scans_feature_flag)
+# or by project
+Feature.disable(:security_on_demand_scans_feature_flag, Project.find(<project id>))
+```
+
+To enable On-demand Scans:
 
 ```ruby
 # Instance-wide
@@ -632,13 +681,29 @@ Feature.enable(:security_on_demand_scans_feature_flag)
 Feature.enable(:security_on_demand_scans_feature_flag, Project.find(<project id>))
 ```
 
-To disable it:
+#### Enable or disable site profiles
+
+The Site Profiles feature is enabled instance-wide by default. You can disable it instance-wide, or disable it
+for specific projects if you prefer.
+[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
+can disable or enable the feature flag.
+
+To disable Site Profiles:
 
 ```ruby
 # Instance-wide
-Feature.disable(:security_on_demand_scans_feature_flag)
+Feature.disable(:security_on_demand_scans_site_profiles_feature_flag)
 # or by project
-Feature.disable(:security_on_demand_scans_feature_flag, Project.find(<project id>))
+Feature.disable(:security_on_demand_scans_site_profiles_feature_flag, Project.find(<project id>))
+```
+
+To enable Site Profiles:
+
+```ruby
+# Instance-wide
+Feature.enable(:security_on_demand_scans_site_profiles_feature_flag)
+# or by project
+Feature.enable(:security_on_demand_scans_site_profiles_feature_flag, Project.find(<project id>))
 ```
 
 ## Reports

doc/user/compliance/license_compliance/index.md

@@ -695,6 +695,16 @@ Additional configuration may be needed for connecting to
 [private Python repositories](#using-private-python-repos),
 and [private Yarn registries](#using-private-yarn-registries).
 
+### SPDX license list name matching
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/212388) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.3.
+
+Prior to GitLab 13.3, offline environments required an exact name match for [project policies](#policies).
+In GitLab 13.3 and later, GitLab matches the name of [project policies](#policies)
+with identifiers from the [SPDX license list](https://spdx.org/licenses/).
+A local copy of the SPDX license list is distributed with the GitLab instance. If needed, the GitLab
+instance's administrator can manually update it with a [Rake task](../../../raketasks/spdx.md).
+
 Exact name matches are required for [project policies](#policies)
 when running in an offline environment ([see related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/212388)).
 

doc/user/group/epics/manage_epics.md

@@ -151,9 +151,18 @@ The sort option and order is saved and used wherever you browse epics, including
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/213068) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.0 behind a feature flag, disabled by default.
 > - [Became enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/224513) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.2.
+> - You can [use the Confidentiality option in the epic sidebar](https://gitlab.com/gitlab-org/gitlab/-/issues/197340) in GitLab [Premium](https://about.gitlab.com/pricing/) 13.3 and later.
 
-When you're creating an epic, you can make it confidential by selecting the **Make this epic
-confidential** checkbox.
+If you're working on items that contain private information, you can make an epic confidential.
+
+NOTE: **Note:**
+A confidential epic can only contain confidential issues and confidential child epics.
+
+To make an epic confidential:
+
+- **When creating an epic:** select the checkbox **Make this epic confidential**.
+- **In an existing epic:** in the epic's sidebar, select **Edit** next to **Confidentiality** then
+  select **Turn on**.
 
 ### Disable confidential epics **(PREMIUM ONLY)**
 

doc/user/search/advanced_search_syntax.md

@@ -73,3 +73,20 @@ Examples:
 - Finding the text 'def create' inside files with the `.rb` extension: `def create extension:rb`
 - Finding the text `sha` inside files in a folder called `encryption`: `sha path:encryption`
 - Finding any file starting with `hello` containing `world` and with the `.js` extension: `world filename:hello* extension:js`
+
+#### Excluding filters
+
+[Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/31684) in GitLab Starter 13.3.
+
+Filters can be inversed to **filter out** results from the result set, by prefixing the filter name with a `-` (hyphen) character, such as:
+
+- `-filename`
+- `-path`
+- `-extension`
+
+Examples:
+
+- Finding `rails` in all files but `Gemfile.lock`: `rails -filename:Gemfile.lock`
+- Finding `success` in all files excluding `.po|pot` files: `success -filename:*.po*`
+- Finding `import` excluding minified JavaScript (`.min.js`) files: `import -extension:min.js`
+- Finding `docs` for all files outside the `docs/` folder: `docs -path:docs/`

lib/api/wikis.rb

@@ -61,9 +61,10 @@ module API
         post ':id/wikis' do
           authorize! :create_wiki, container
 
-          page = WikiPages::CreateService.new(container: container, current_user: current_user, params: params).execute
+          response = WikiPages::CreateService.new(container: container, current_user: current_user, params: params).execute
+          page = response.payload[:page]
 
-          if page.valid?
+          if response.success?
             present page, with: Entities::WikiPage
           else
             render_validation_error!(page)

lib/gitlab/database.rb

@@ -22,6 +22,7 @@ module Gitlab
 
     # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
     MAX_INT_VALUE = 2147483647
+    MIN_INT_VALUE = -2147483648
 
     # The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz:
     # https://www.postgresql.org/docs/9.1/static/datatype-datetime.html

lib/gitlab/database/with_lock_retries.rb

@@ -2,7 +2,14 @@
 
 module Gitlab
   module Database
+    # This class provides a way to automatically execute code that relies on acquiring a database lock in a way
+    # designed to minimize impact on a busy production database.
+    #
+    # A default timing configuration is provided that makes repeated attempts to acquire the necessary lock, with
+    # varying lock_timeout settings, and also serves to limit the maximum number of attempts.
     class WithLockRetries
+      AttemptsExhaustedError = Class.new(StandardError)
+
       NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null')
 
       # Each element of the array represents a retry iteration.
@@ -63,7 +70,17 @@ module Gitlab
         @log_params = { method: 'with_lock_retries', class: klass.to_s }
       end
 
-      def run(&block)
+      # Executes a block of code, retrying it whenever a database lock can't be acquired in time
+      #
+      # When a database lock can't be acquired, ActiveRecord throws ActiveRecord::LockWaitTimeout
+      # exception which we intercept to re-execute the block of code, until it finishes or we reach the
+      # max attempt limit. The default behavior when max attempts have been reached is to make a final attempt with the
+      # lock_timeout disabled, but this can be altered with the raise_on_exhaustion parameter.
+      #
+      # @see DEFAULT_TIMING_CONFIGURATION for the timings used when attempting a retry
+      # @param [Boolean] raise_on_exhaustion whether to raise `AttemptsExhaustedError` when exhausting max attempts
+      # @param [Proc] block of code that will be executed
+      def run(raise_on_exhaustion: false, &block)
         raise 'no block given' unless block_given?
 
         @block = block
@@ -85,6 +102,9 @@ module Gitlab
             retry
           else
             reset_db_settings
+
+            raise AttemptsExhaustedError, 'configured attempts to obtain locks are exhausted' if raise_on_exhaustion
+
             run_block_without_lock_timeout
           end
 

lib/gitlab/reactive_cache_set_cache.rb

@@ -20,7 +20,7 @@ module Gitlab
         keys << cache_key(key)
 
         redis.pipelined do
-          keys.each_slice(1000) { |subset| redis.del(*subset) }
+          keys.each_slice(1000) { |subset| redis.unlink(*subset) }
         end
       end
     end

lib/gitlab/search/parsed_query.rb

@@ -3,6 +3,8 @@
 module Gitlab
   module Search
     class ParsedQuery
+      include Gitlab::Utils::StrongMemoize
+
       attr_reader :term, :filters
 
       def initialize(term, filters)
@@ -11,13 +13,44 @@ module Gitlab
       end
 
       def filter_results(results)
-        filters = @filters.reject { |filter| filter[:matcher].nil? }
-        return unless filters
+        with_matcher = ->(filter) { filter[:matcher].present? }
 
-        results.select do |result|
-          filters.all? do |filter|
-            filter[:matcher].call(filter, result)
-          end
+        excluding = excluding_filters.select(&with_matcher)
+        including = including_filters.select(&with_matcher)
+
+        return unless excluding.any? || including.any?
+
+        results.select! do |result|
+          including.all? { |filter| filter[:matcher].call(filter, result) }
+        end
+
+        results.reject! do |result|
+          excluding.any? { |filter| filter[:matcher].call(filter, result) }
+        end
+
+        results
+      end
+
+      private
+
+      def including_filters
+        processed_filters(:including)
+      end
+
+      def excluding_filters
+        processed_filters(:excluding)
+      end
+
+      def processed_filters(type)
+        excluding, including = strong_memoize(:processed_filters) do
+          filters.partition { |filter| filter[:negated] }
+        end
+
+        case type
+        when :including then including
+        when :excluding then excluding
+        else
+          raise ArgumentError.new(type)
         end
       end
     end

lib/gitlab/search/query.rb

@@ -20,7 +20,10 @@ module Gitlab
       private
 
       def filter(name, **attributes)
-        filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes)
+        filter = {
+          parser: @filter_options[:default_parser],
+          name: name
+        }.merge(attributes)
 
         @filters << filter
       end
@@ -33,12 +36,13 @@ module Gitlab
         fragments = []
 
         filters = @filters.each_with_object([]) do |filter, parsed_filters|
-          match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ }
+          match = @raw_query.split.find { |part| part =~ /\A-?#{filter[:name]}:/ }
           next unless match
 
           input = match.split(':')[1..-1].join
           next if input.empty?
 
+          filter[:negated] = match.start_with?("-")
           filter[:value] = parse_filter(filter, input)
           filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?')
           fragments << match

locale/gitlab.pot

@@ -6934,6 +6934,9 @@ msgstr ""
 msgid "Could not create project"
 msgstr ""
 
+msgid "Could not create wiki page"
+msgstr ""
+
 msgid "Could not delete %{design}. Please try again."
 msgstr ""
 

spec/features/boards/issue_ordering_spec.rb

@@ -21,7 +21,7 @@ RSpec.describe 'Issue Boards', :js do
   end
 
   context 'un-ordered issues' do
-    let!(:issue4) { create(:labeled_issue, project: project, labels: [label]) }
+    let!(:issue4) { create(:labeled_issue, project: project, labels: [label], relative_position: nil) }
 
     before do
       visit project_board_path(project, board)

spec/features/projects/settings/operations_settings_spec.rb

@@ -46,11 +46,14 @@ RSpec.describe 'Projects > Settings > For a forked project', :js do
       it 'updates form values' do
         check(create_issue)
         uncheck(send_email)
+        click_on('No template selected')
+        click_on('bug')
 
         save_form
         click_expand_incident_management_button
 
         expect(find_field(create_issue)).to be_checked
+        expect(page).to have_selector(:id, 'alert-integration-settings-issue-template', text: 'bug')
         expect(find_field(send_email)).not_to be_checked
       end
 

spec/lib/gitlab/danger/helper_spec.rb

@@ -236,7 +236,6 @@ RSpec.describe Gitlab::Danger::Helper do
 
       'generator_templates/foo' | [:backend]
       'vendor/languages.yml'    | [:backend]
-      'vendor/licenses.csv'     | [:backend]
       'file_hooks/examples/'    | [:backend]
 
       'Gemfile'        | [:backend]

spec/lib/gitlab/database/with_lock_retries_spec.rb

@@ -72,9 +72,14 @@ RSpec.describe Gitlab::Database::WithLockRetries do
           lock_attempts = 0
           lock_acquired = false
 
-          expect_any_instance_of(Gitlab::Database::WithLockRetries).to receive(:sleep).exactly(retry_count - 1).times # we don't sleep in the last iteration
+          # the actual number of attempts to run_block_with_transaction can never exceed the number of
+          # timings_configurations, so here we limit the retry_count if it exceeds that value
+          #
+          # also, there is no call to sleep after the final attempt, which is why it will always be one less
+          expected_runs_with_timeout = [retry_count, timing_configuration.size].min
+          expect(subject).to receive(:sleep).exactly(expected_runs_with_timeout - 1).times
 
-          allow_any_instance_of(Gitlab::Database::WithLockRetries).to receive(:run_block_with_transaction).and_wrap_original do |method|
+          expect(subject).to receive(:run_block_with_transaction).exactly(expected_runs_with_timeout).times.and_wrap_original do |method|
             lock_fiber.resume if lock_attempts == retry_count
 
             method.call
@@ -114,6 +119,33 @@ RSpec.describe Gitlab::Database::WithLockRetries do
         end
       end
 
+      context 'after the retries, when requested to raise an error' do
+        let(:expected_attempts_with_timeout) { timing_configuration.size }
+        let(:retry_count) { timing_configuration.size + 1 }
+
+        it 'raises an error instead of waiting indefinitely for the lock' do
+          lock_attempts = 0
+          lock_acquired = false
+
+          expect(subject).to receive(:sleep).exactly(expected_attempts_with_timeout - 1).times
+          expect(subject).to receive(:run_block_with_transaction).exactly(expected_attempts_with_timeout).times.and_call_original
+
+          expect do
+            subject.run(raise_on_exhaustion: true) do
+              lock_attempts += 1
+
+              ActiveRecord::Base.transaction do
+                ActiveRecord::Base.connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode")
+                lock_acquired = true
+              end
+            end
+          end.to raise_error(described_class::AttemptsExhaustedError)
+
+          expect(lock_attempts).to eq(retry_count - 1)
+          expect(lock_acquired).to eq(false)
+        end
+      end
+
       context 'when statement timeout is reached' do
         it 'raises QueryCanceled error' do
           lock_acquired = false

spec/lib/gitlab/file_finder_spec.rb

@@ -13,22 +13,44 @@ RSpec.describe Gitlab::FileFinder do
       let(:expected_file_by_content) { 'CHANGELOG' }
     end
 
-    it 'filters by filename' do
-      results = subject.find('files filename:wm.svg')
+    context 'with inclusive filters' do
+      it 'filters by filename' do
+        results = subject.find('files filename:wm.svg')
 
-      expect(results.count).to eq(1)
+        expect(results.count).to eq(1)
+      end
+
+      it 'filters by path' do
+        results = subject.find('white path:images')
+
+        expect(results.count).to eq(1)
+      end
+
+      it 'filters by extension' do
+        results = subject.find('files extension:md')
+
+        expect(results.count).to eq(4)
+      end
     end
 
-    it 'filters by path' do
-      results = subject.find('white path:images')
+    context 'with exclusive filters' do
+      it 'filters by filename' do
+        results = subject.find('files -filename:wm.svg')
 
-      expect(results.count).to eq(1)
-    end
+        expect(results.count).to eq(26)
+      end
 
-    it 'filters by extension' do
-      results = subject.find('files extension:svg')
+      it 'filters by path' do
+        results = subject.find('white -path:images')
 
-      expect(results.count).to eq(1)
+        expect(results.count).to eq(4)
+      end
+
+      it 'filters by extension' do
+        results = subject.find('files -extension:md')
+
+        expect(results.count).to eq(23)
+      end
     end
 
     it 'does not cause N+1 query' do

spec/lib/gitlab/search/query_spec.rb

@@ -38,4 +38,12 @@ RSpec.describe Gitlab::Search::Query do
       expect(subject.term).to eq(query)
     end
   end
+
+  context 'with an exclusive filter' do
+    let(:query) { 'something -name:bingo -other:dingo' }
+
+    it 'negates the filter' do
+      expect(subject.filters).to all(include(negated: true))
+    end
+  end
 end

spec/models/analytics/cycle_analytics/project_stage_spec.rb

@@ -23,7 +23,7 @@ RSpec.describe Analytics::CycleAnalytics::ProjectStage do
 
   context 'relative positioning' do
     it_behaves_like 'a class that supports relative positioning' do
-      let(:project) { build(:project) }
+      let_it_be(:project) { create(:project) }
       let(:factory) { :cycle_analytics_project_stage }
       let(:default_params) { { project: project } }
     end

spec/models/issue_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe Issue do
   include ExternalAuthorizationServiceHelpers
 
   let_it_be(:user) { create(:user) }
+  let_it_be(:reusable_project) { create(:project) }
 
   describe "Associations" do
     it { is_expected.to belong_to(:milestone) }
@@ -145,13 +146,13 @@ RSpec.describe Issue do
   end
 
   describe '#order_by_position_and_priority' do
-    let(:project) { create :project }
+    let(:project) { reusable_project }
     let(:p1) { create(:label, title: 'P1', project: project, priority: 1) }
     let(:p2) { create(:label, title: 'P2', project: project, priority: 2) }
     let!(:issue1) { create(:labeled_issue, project: project, labels: [p1]) }
     let!(:issue2) { create(:labeled_issue, project: project, labels: [p2]) }
-    let!(:issue3) { create(:issue, project: project, relative_position: 100) }
-    let!(:issue4) { create(:issue, project: project, relative_position: 200) }
+    let!(:issue3) { create(:issue, project: project, relative_position: -200) }
+    let!(:issue4) { create(:issue, project: project, relative_position: -100) }
 
     it 'returns ordered list' do
       expect(project.issues.order_by_position_and_priority)
@@ -160,10 +161,10 @@ RSpec.describe Issue do
   end
 
   describe '#sort' do
-    let(:project) { create(:project) }
+    let(:project) { reusable_project }
 
     context "by relative_position" do
-      let!(:issue)  { create(:issue, project: project) }
+      let!(:issue)  { create(:issue, project: project, relative_position: nil) }
       let!(:issue2) { create(:issue, project: project, relative_position: 2) }
       let!(:issue3) { create(:issue, project: project, relative_position: 1) }
 
@@ -1027,7 +1028,7 @@ RSpec.describe Issue do
 
   context "relative positioning" do
     it_behaves_like "a class that supports relative positioning" do
-      let(:project) { create(:project) }
+      let_it_be(:project) { create(:project) }
       let(:factory) { :issue }
       let(:default_params) { { project: project } }
     end

spec/support/shared_examples/controllers/wiki_actions_shared_examples.rb

@@ -355,6 +355,44 @@ RSpec.shared_examples 'wiki controller actions' do
     end
   end
 
+  describe 'POST #create' do
+    let(:new_title) { 'New title' }
+    let(:new_content) { 'New content' }
+
+    subject do
+      post(:create,
+            params: routing_params.merge(
+              wiki: { title: new_title, content: new_content }
+            ))
+    end
+
+    context 'when page is valid' do
+      it 'creates the page' do
+        expect do
+          subject
+        end.to change { wiki.list_pages.size }.by 1
+
+        wiki_page = wiki.find_page(new_title)
+
+        expect(wiki_page.title).to eq new_title
+        expect(wiki_page.content).to eq new_content
+      end
+    end
+
+    context 'when page is not valid' do
+      let(:new_title) { '' }
+
+      it 'renders the edit state' do
+        expect do
+          subject
+        end.not_to change { wiki.list_pages.size }
+
+        expect(response).to render_template('shared/wikis/edit')
+        expect(flash[:alert]).to eq('Could not create wiki page')
+      end
+    end
+  end
+
   def redirect_to_wiki(wiki, page)
     redirect_to(controller.wiki_page_path(wiki, page))
   end

spec/support/shared_examples/models/relative_positioning_shared_examples.rb

@@ -25,7 +25,6 @@ RSpec.shared_examples 'a class that supports relative positioning' do
 
       items = [item1, item2, item3]
       described_class.move_nulls_to_end(items)
-      items.map(&:reload)
 
       expect(items.sort_by(&:relative_position)).to eq(items)
       expect(item1.relative_position).to be(1000)
@@ -35,22 +34,57 @@ RSpec.shared_examples 'a class that supports relative positioning' do
       expect(item3.next_relative_position).to be_nil
     end
 
+    it 'preserves relative position' do
+      item1.update!(relative_position: nil)
+      item2.update!(relative_position: nil)
+
+      described_class.move_nulls_to_end([item1, item2])
+
+      expect(item1.relative_position).to be < item2.relative_position
+    end
+
     it 'moves the item near the start position when there are no existing positions' do
       item1.update!(relative_position: nil)
 
       described_class.move_nulls_to_end([item1])
-      item1.reload
-
-      expect(item1.relative_position).to eq(described_class::START_POSITION + described_class::IDEAL_DISTANCE)
+      expect(item1.reset.relative_position).to eq(described_class::START_POSITION + described_class::IDEAL_DISTANCE)
     end
 
     it 'does not perform any moves if all items have their relative_position set' do
       item1.update!(relative_position: 1)
 
-      described_class.move_nulls_to_end([item1])
-      item1.reload
+      expect do
+        described_class.move_nulls_to_end([item1])
+      end.not_to change { item1.reset.relative_position }
+    end
 
-      expect(item1.relative_position).to be(1)
+    it 'manages to move nulls to the end even if there is a sequence at the end' do
+      bunch = create_items_with_positions(run_at_end)
+      item1.update!(relative_position: nil)
+
+      described_class.move_nulls_to_end([item1])
+
+      items = [*bunch, item1]
+      items.each(&:reset)
+
+      expect(items.map(&:relative_position)).to all(be_valid_position)
+      expect(items.sort_by(&:relative_position)).to eq(items)
+    end
+
+    it 'does not have an N+1 issue' do
+      create_items_with_positions(10..12)
+
+      a, b, c, d, e, f = create_items_with_positions([nil, nil, nil, nil, nil, nil])
+
+      baseline = ActiveRecord::QueryRecorder.new do
+        described_class.move_nulls_to_end([a, e])
+      end
+
+      expect { described_class.move_nulls_to_end([b, c, d]) }
+        .not_to exceed_query_limit(baseline)
+
+      expect { described_class.move_nulls_to_end([f]) }
+        .not_to exceed_query_limit(baseline.count)
     end
   end
 
@@ -78,11 +112,19 @@ RSpec.shared_examples 'a class that supports relative positioning' do
       item1.update!(relative_position: nil)
 
       described_class.move_nulls_to_start([item1])
-      item1.reload
 
       expect(item1.relative_position).to eq(described_class::START_POSITION - described_class::IDEAL_DISTANCE)
     end
 
+    it 'preserves relative position' do
+      item1.update!(relative_position: nil)
+      item2.update!(relative_position: nil)
+
+      described_class.move_nulls_to_end([item1, item2])
+
+      expect(item1.relative_position).to be < item2.relative_position
+    end
+
     it 'does not perform any moves if all items have their relative_position set' do
       item1.update!(relative_position: 1)
 
@@ -101,8 +143,8 @@ RSpec.shared_examples 'a class that supports relative positioning' do
 
   describe '#prev_relative_position' do
     it 'returns previous position if there is an item above' do
-      item1.update(relative_position: 5)
-      item2.update(relative_position: 15)
+      item1.update!(relative_position: 5)
+      item2.update!(relative_position: 15)
 
       expect(item2.prev_relative_position).to eq item1.relative_position
     end
@@ -114,8 +156,8 @@ RSpec.shared_examples 'a class that supports relative positioning' do
 
   describe '#next_relative_position' do
     it 'returns next position if there is an item below' do
-      item1.update(relative_position: 5)
-      item2.update(relative_position: 15)
+      item1.update!(relative_position: 5)
+      item2.update!(relative_position: 15)
 
       expect(item1.next_relative_position).to eq item2.relative_position
     end
@@ -125,9 +167,172 @@ RSpec.shared_examples 'a class that supports relative positioning' do
     end
   end
 
+  describe '#find_next_gap_before' do
+    context 'there is no gap' do
+      let(:items) { create_items_with_positions(run_at_start) }
+
+      it 'returns nil' do
+        items.each do |item|
+          expect(item.send(:find_next_gap_before)).to be_nil
+        end
+      end
+    end
+
+    context 'there is a sequence ending at MAX_POSITION' do
+      let(:items) { create_items_with_positions(run_at_end) }
+
+      let(:gaps) do
+        items.map { |item| item.send(:find_next_gap_before) }
+      end
+
+      it 'can find the gap at the start for any item in the sequence' do
+        gap = { start: items.first.relative_position, end: RelativePositioning::MIN_POSITION }
+
+        expect(gaps).to all(eq(gap))
+      end
+
+      it 'respects lower bounds' do
+        gap = { start: items.first.relative_position, end: 10 }
+        new_item.update!(relative_position: 10)
+
+        expect(gaps).to all(eq(gap))
+      end
+    end
+
+    specify do
+      item1.update!(relative_position: 5)
+
+      (0..10).each do |pos|
+        item2.update!(relative_position: pos)
+
+        gap = item2.send(:find_next_gap_before)
+
+        expect(gap[:start]).to be <= item2.relative_position
+        expect((gap[:end] - gap[:start]).abs).to be >= RelativePositioning::MIN_GAP
+        expect(gap[:start]).to be_valid_position
+        expect(gap[:end]).to be_valid_position
+      end
+    end
+
+    it 'deals with there not being any items to the left' do
+      create_items_with_positions([1, 2, 3])
+      new_item.update!(relative_position: 0)
+
+      expect(new_item.send(:find_next_gap_before)).to eq(start: 0, end: RelativePositioning::MIN_POSITION)
+    end
+
+    it 'finds the next gap to the left, skipping adjacent values' do
+      create_items_with_positions([1, 9, 10])
+      new_item.update!(relative_position: 11)
+
+      expect(new_item.send(:find_next_gap_before)).to eq(start: 9, end: 1)
+    end
+
+    it 'finds the next gap to the left' do
+      create_items_with_positions([2, 10])
+
+      new_item.update!(relative_position: 15)
+      expect(new_item.send(:find_next_gap_before)).to eq(start: 15, end: 10)
+
+      new_item.update!(relative_position: 11)
+      expect(new_item.send(:find_next_gap_before)).to eq(start: 10, end: 2)
+
+      new_item.update!(relative_position: 9)
+      expect(new_item.send(:find_next_gap_before)).to eq(start: 9, end: 2)
+
+      new_item.update!(relative_position: 5)
+      expect(new_item.send(:find_next_gap_before)).to eq(start: 5, end: 2)
+    end
+  end
+
+  describe '#find_next_gap_after' do
+    context 'there is no gap' do
+      let(:items) { create_items_with_positions(run_at_end) }
+
+      it 'returns nil' do
+        items.each do |item|
+          expect(item.send(:find_next_gap_after)).to be_nil
+        end
+      end
+    end
+
+    context 'there is a sequence starting at MIN_POSITION' do
+      let(:items) { create_items_with_positions(run_at_start) }
+
+      let(:gaps) do
+        items.map { |item| item.send(:find_next_gap_after) }
+      end
+
+      it 'can find the gap at the end for any item in the sequence' do
+        gap = { start: items.last.relative_position, end: RelativePositioning::MAX_POSITION }
+
+        expect(gaps).to all(eq(gap))
+      end
+
+      it 'respects upper bounds' do
+        gap = { start: items.last.relative_position, end: 10 }
+        new_item.update!(relative_position: 10)
+
+        expect(gaps).to all(eq(gap))
+      end
+    end
+
+    specify do
+      item1.update!(relative_position: 5)
+
+      (0..10).each do |pos|
+        item2.update!(relative_position: pos)
+
+        gap = item2.send(:find_next_gap_after)
+
+        expect(gap[:start]).to be >= item2.relative_position
+        expect((gap[:end] - gap[:start]).abs).to be >= RelativePositioning::MIN_GAP
+        expect(gap[:start]).to be_valid_position
+        expect(gap[:end]).to be_valid_position
+      end
+    end
+
+    it 'deals with there not being any items to the right' do
+      create_items_with_positions([1, 2, 3])
+      new_item.update!(relative_position: 5)
+
+      expect(new_item.send(:find_next_gap_after)).to eq(start: 5, end: RelativePositioning::MAX_POSITION)
+    end
+
+    it 'finds the next gap to the right, skipping adjacent values' do
+      create_items_with_positions([1, 2, 10])
+      new_item.update!(relative_position: 0)
+
+      expect(new_item.send(:find_next_gap_after)).to eq(start: 2, end: 10)
+    end
+
+    it 'finds the next gap to the right' do
+      create_items_with_positions([2, 10])
+
+      new_item.update!(relative_position: 0)
+      expect(new_item.send(:find_next_gap_after)).to eq(start: 0, end: 2)
+
+      new_item.update!(relative_position: 1)
+      expect(new_item.send(:find_next_gap_after)).to eq(start: 2, end: 10)
+
+      new_item.update!(relative_position: 3)
+      expect(new_item.send(:find_next_gap_after)).to eq(start: 3, end: 10)
+
+      new_item.update!(relative_position: 5)
+      expect(new_item.send(:find_next_gap_after)).to eq(start: 5, end: 10)
+    end
+  end
+
   describe '#move_before' do
+    let(:item3) { create(factory, default_params) }
+
     it 'moves item before' do
-      [item2, item1].each(&:move_to_end)
+      [item2, item1].each do |item|
+        item.move_to_end
+        item.save!
+      end
+
+      expect(item1.relative_position).to be > item2.relative_position
 
       item1.move_before(item2)
 
@@ -135,12 +340,10 @@ RSpec.shared_examples 'a class that supports relative positioning' do
     end
 
     context 'when there is no space' do
-      let(:item3) { create(factory, default_params) }
-
       before do
-        item1.update(relative_position: 1000)
-        item2.update(relative_position: 1001)
-        item3.update(relative_position: 1002)
+        item1.update!(relative_position: 1000)
+        item2.update!(relative_position: 1001)
+        item3.update!(relative_position: 1002)
       end
 
       it 'moves items correctly' do
@@ -149,6 +352,73 @@ RSpec.shared_examples 'a class that supports relative positioning' do
         expect(item3.relative_position).to be_between(item1.reload.relative_position, item2.reload.relative_position).exclusive
       end
     end
+
+    it 'can move the item before an item at the start' do
+      item1.update!(relative_position: RelativePositioning::START_POSITION)
+
+      new_item.move_before(item1)
+
+      expect(new_item.relative_position).to be_valid_position
+      expect(new_item.relative_position).to be < item1.reload.relative_position
+    end
+
+    it 'can move the item before an item at MIN_POSITION' do
+      item1.update!(relative_position: RelativePositioning::MIN_POSITION)
+
+      new_item.move_before(item1)
+
+      expect(new_item.relative_position).to be >= RelativePositioning::MIN_POSITION
+      expect(new_item.relative_position).to be < item1.reload.relative_position
+    end
+
+    it 'can move the item before an item bunched up at MIN_POSITION' do
+      item1, item2, item3 = create_items_with_positions(run_at_start)
+
+      new_item.move_before(item3)
+      new_item.save!
+
+      items = [item1, item2, new_item, item3]
+
+      items.each do |item|
+        expect(item.reset.relative_position).to be_valid_position
+      end
+
+      expect(items.sort_by(&:relative_position)).to eq(items)
+    end
+
+    context 'leap-frogging to the left' do
+      before do
+        start = RelativePositioning::START_POSITION
+        item1.update!(relative_position: start - RelativePositioning::IDEAL_DISTANCE * 0)
+        item2.update!(relative_position: start - RelativePositioning::IDEAL_DISTANCE * 1)
+        item3.update!(relative_position: start - RelativePositioning::IDEAL_DISTANCE * 2)
+      end
+
+      let(:item3) { create(factory, default_params) }
+
+      def leap_frog(steps)
+        a = item1
+        b = item2
+
+        steps.times do |i|
+          a.move_before(b)
+          a.save!
+          a, b = b, a
+        end
+      end
+
+      it 'can leap-frog STEPS - 1 times before needing to rebalance' do
+        # This is less efficient than going right, due to the flooring of
+        # integer division
+        expect { leap_frog(RelativePositioning::STEPS - 1) }
+          .not_to change { item3.reload.relative_position }
+      end
+
+      it 'rebalances after leap-frogging STEPS times' do
+        expect { leap_frog(RelativePositioning::STEPS) }
+          .to change { item3.reload.relative_position }
+      end
+    end
   end
 
   describe '#move_after' do
@@ -164,9 +434,17 @@ RSpec.shared_examples 'a class that supports relative positioning' do
       let(:item3) { create(factory, default_params) }
 
       before do
-        item1.update(relative_position: 1000)
-        item2.update(relative_position: 1001)
-        item3.update(relative_position: 1002)
+        item1.update!(relative_position: 1000)
+        item2.update!(relative_position: 1001)
+        item3.update!(relative_position: 1002)
+      end
+
+      it 'can move the item after an item at MAX_POSITION' do
+        item1.update!(relative_position: RelativePositioning::MAX_POSITION)
+
+        new_item.move_after(item1)
+        expect(new_item.relative_position).to be_valid_position
+        expect(new_item.relative_position).to be > item1.reset.relative_position
       end
 
       it 'moves items correctly' do
@@ -175,6 +453,53 @@ RSpec.shared_examples 'a class that supports relative positioning' do
         expect(item1.relative_position).to be_between(item2.reload.relative_position, item3.reload.relative_position).exclusive
       end
     end
+
+    it 'can move the item after an item bunched up at MAX_POSITION' do
+      item1, item2, item3 = create_items_with_positions(run_at_end)
+
+      new_item.move_after(item1)
+      new_item.save!
+
+      items = [item1, new_item, item2, item3]
+
+      items.each do |item|
+        expect(item.reset.relative_position).to be_valid_position
+      end
+
+      expect(items.sort_by(&:relative_position)).to eq(items)
+    end
+
+    context 'leap-frogging' do
+      before do
+        start = RelativePositioning::START_POSITION
+        item1.update!(relative_position: start + RelativePositioning::IDEAL_DISTANCE * 0)
+        item2.update!(relative_position: start + RelativePositioning::IDEAL_DISTANCE * 1)
+        item3.update!(relative_position: start + RelativePositioning::IDEAL_DISTANCE * 2)
+      end
+
+      let(:item3) { create(factory, default_params) }
+
+      def leap_frog(steps)
+        a = item1
+        b = item2
+
+        steps.times do |i|
+          a.move_after(b)
+          a.save!
+          a, b = b, a
+        end
+      end
+
+      it 'can leap-frog STEPS times before needing to rebalance' do
+        expect { leap_frog(RelativePositioning::STEPS) }
+          .not_to change { item3.reload.relative_position }
+      end
+
+      it 'rebalances after leap-frogging STEPS+1 times' do
+        expect { leap_frog(RelativePositioning::STEPS + 1) }
+          .to change { item3.reload.relative_position }
+      end
+    end
   end
 
   describe '#move_to_end' do
@@ -193,8 +518,17 @@ RSpec.shared_examples 'a class that supports relative positioning' do
 
   describe '#move_between' do
     before do
-      [item1, item2].each do |item1|
-        item1.move_to_end && item1.save
+      [item1, item2].each do |item|
+        item.move_to_end && item.save!
+      end
+    end
+
+    shared_examples 'moves item between' do
+      it 'moves the middle item to between left and right' do
+        expect do
+          middle.move_between(left, right)
+          middle.save!
+        end.to change { between_exclusive?(left, middle, right) }.from(false).to(true)
       end
     end
 
@@ -218,26 +552,26 @@ RSpec.shared_examples 'a class that supports relative positioning' do
     end
 
     it 'positions items even when after and before positions are the same' do
-      item2.update relative_position: item1.relative_position
+      item2.update! relative_position: item1.relative_position
 
       new_item.move_between(item1, item2)
+      [item1, item2].each(&:reset)
 
       expect(new_item.relative_position).to be > item1.relative_position
       expect(item1.relative_position).to be < item2.relative_position
     end
 
-    it 'positions items between other two if distance is 1' do
-      item2.update relative_position: item1.relative_position + 1
+    context 'the two items are next to each other' do
+      let(:left) { item1 }
+      let(:middle) { new_item }
+      let(:right) { create_item(relative_position: item1.relative_position + 1) }
 
-      new_item.move_between(item1, item2)
-
-      expect(new_item.relative_position).to be > item1.relative_position
-      expect(item1.relative_position).to be < item2.relative_position
+      it_behaves_like 'moves item between'
     end
 
     it 'positions item in the middle of other two if distance is big enough' do
-      item1.update relative_position: 6000
-      item2.update relative_position: 10000
+      item1.update! relative_position: 6000
+      item2.update! relative_position: 10000
 
       new_item.move_between(item1, item2)
 
@@ -245,7 +579,8 @@ RSpec.shared_examples 'a class that supports relative positioning' do
     end
 
     it 'positions item closer to the middle if we are at the very top' do
-      item2.update relative_position: 6000
+      item1.update!(relative_position: 6001)
+      item2.update!(relative_position: 6000)
 
       new_item.move_between(nil, item2)
 
@@ -253,51 +588,53 @@ RSpec.shared_examples 'a class that supports relative positioning' do
     end
 
     it 'positions item closer to the middle if we are at the very bottom' do
-      new_item.update relative_position: 1
-      item1.update relative_position: 6000
-      item2.destroy
+      new_item.update!(relative_position: 1)
+      item1.update!(relative_position: 6000)
+      item2.update!(relative_position: 5999)
 
       new_item.move_between(item1, nil)
 
       expect(new_item.relative_position).to eq(6000 + RelativePositioning::IDEAL_DISTANCE)
     end
 
-    it 'positions item in the middle of other two if distance is not big enough' do
-      item1.update relative_position: 100
-      item2.update relative_position: 400
+    it 'positions item in the middle of other two' do
+      item1.update! relative_position: 100
+      item2.update! relative_position: 400
 
       new_item.move_between(item1, item2)
 
       expect(new_item.relative_position).to eq(250)
     end
 
-    it 'positions item in the middle of other two is there is no place' do
-      item1.update relative_position: 100
-      item2.update relative_position: 101
+    context 'there is no space' do
+      let(:middle) { new_item }
+      let(:left) { create_item(relative_position: 100) }
+      let(:right) { create_item(relative_position: 101) }
 
-      new_item.move_between(item1, item2)
-
-      expect(new_item.relative_position).to be_between(item1.relative_position, item2.relative_position).exclusive
+      it_behaves_like 'moves item between'
     end
 
-    it 'uses rebalancing if there is no place' do
-      item1.update relative_position: 100
-      item2.update relative_position: 101
-      item3 = create_item(relative_position: 102)
-      new_item.update relative_position: 103
+    context 'there is a bunch of items' do
+      let(:items) { create_items_with_positions(100..104) }
+      let(:left) { items[1] }
+      let(:middle) { items[3] }
+      let(:right) { items[2] }
 
-      new_item.move_between(item2, item3)
-      new_item.save!
+      it_behaves_like 'moves item between'
 
-      expect(new_item.relative_position).to be_between(item2.relative_position, item3.relative_position).exclusive
-      expect(item1.reload.relative_position).not_to eq(100)
+      it 'handles bunches correctly' do
+        middle.move_between(left, right)
+        middle.save!
+
+        expect(items.first.reset.relative_position).to be < middle.relative_position
+      end
     end
 
-    it 'positions item right if we pass none-sequential parameters' do
-      item1.update relative_position: 99
-      item2.update relative_position: 101
+    it 'positions item right if we pass non-sequential parameters' do
+      item1.update! relative_position: 99
+      item2.update! relative_position: 101
       item3 = create_item(relative_position: 102)
-      new_item.update relative_position: 103
+      new_item.update! relative_position: 103
 
       new_item.move_between(item1, item3)
       new_item.save!
@@ -329,6 +666,12 @@ RSpec.shared_examples 'a class that supports relative positioning' do
       expect(positions).to eq([90, 95, 96, 102])
     end
 
+    it 'raises an error if there is no space' do
+      items = create_items_with_positions(run_at_start)
+
+      expect { items.last.move_sequence_before }.to raise_error(RelativePositioning::NoSpaceLeft)
+    end
+
     it 'finds a gap if there are unused positions' do
       items = create_items_with_positions([100, 101, 102])
 
@@ -336,7 +679,8 @@ RSpec.shared_examples 'a class that supports relative positioning' do
       items.last.save!
 
       positions = items.map { |item| item.reload.relative_position }
-      expect(positions).to eq([50, 51, 102])
+
+      expect(positions.last - positions.second).to be > RelativePositioning::MIN_GAP
     end
   end
 
@@ -358,7 +702,33 @@ RSpec.shared_examples 'a class that supports relative positioning' do
       items.first.save!
 
       positions = items.map { |item| item.reload.relative_position }
-      expect(positions).to eq([100, 601, 602])
+      expect(positions.second - positions.first).to be > RelativePositioning::MIN_GAP
+    end
+
+    it 'raises an error if there is no space' do
+      items = create_items_with_positions(run_at_end)
+
+      expect { items.first.move_sequence_after }.to raise_error(RelativePositioning::NoSpaceLeft)
     end
   end
+
+  def be_valid_position
+    be_between(RelativePositioning::MIN_POSITION, RelativePositioning::MAX_POSITION)
+  end
+
+  def between_exclusive?(left, middle, right)
+    a, b, c = [left, middle, right].map { |item| item.reset.relative_position }
+    return false if a.nil? || b.nil?
+    return a < b if c.nil?
+
+    a < b && b < c
+  end
+
+  def run_at_end(size = 3)
+    (RelativePositioning::MAX_POSITION - size)..RelativePositioning::MAX_POSITION
+  end
+
+  def run_at_start(size = 3)
+    (RelativePositioning::MIN_POSITION..).take(size)
+  end
 end

spec/support/shared_examples/services/wiki_pages/create_service_shared_examples.rb

@@ -16,8 +16,10 @@ RSpec.shared_examples 'WikiPages::CreateService#execute' do |container_type|
   subject(:service) { described_class.new(container: container, current_user: user, params: opts) }
 
   it 'creates wiki page with valid attributes' do
-    page = service.execute
+    response = service.execute
+    page = response.payload[:page]
 
+    expect(response).to be_success
     expect(page).to be_valid
     expect(page).to be_persisted
     expect(page.title).to eq(opts[:title])
@@ -77,7 +79,12 @@ RSpec.shared_examples 'WikiPages::CreateService#execute' do |container_type|
     end
 
     it 'reports the error' do
-      expect(service.execute).to be_invalid
+      response = service.execute
+      page = response.payload[:page]
+
+      expect(response).to be_error
+
+      expect(page).to be_invalid
         .and have_attributes(errors: be_present)
     end
   end

spec/views/profiles/preferences/show.html.haml_spec.rb

@@ -12,6 +12,26 @@ RSpec.describe 'profiles/preferences/show' do
     allow(controller).to receive(:current_user).and_return(user)
   end
 
+  context 'navigation theme' do
+    before do
+      render
+    end
+
+    it 'has an id for anchoring' do
+      expect(rendered).to have_css('#navigation-theme')
+    end
+  end
+
+  context 'syntax highlighting theme' do
+    before do
+      render
+    end
+
+    it 'has an id for anchoring' do
+      expect(rendered).to have_css('#syntax-highlighting-theme')
+    end
+  end
+
   context 'behavior' do
     before do
       render
@@ -21,7 +41,7 @@ RSpec.describe 'profiles/preferences/show' do
       expect(rendered).to have_unchecked_field('Render whitespace characters in the Web IDE')
     end
 
-    it 'has an id for anchoring on behavior' do
+    it 'has an id for anchoring' do
       expect(rendered).to have_css('#behavior')
     end
 
@@ -31,13 +51,23 @@ RSpec.describe 'profiles/preferences/show' do
     end
   end
 
+  context 'localization' do
+    before do
+      render
+    end
+
+    it 'has an id for anchoring' do
+      expect(rendered).to have_css('#localization')
+    end
+  end
+
   context 'sourcegraph' do
     def have_sourcegraph_field(*args)
       have_field('user_sourcegraph_enabled', *args)
     end
 
     def have_integrations_section
-      have_css('.profile-settings-sidebar', { text: 'Integrations' })
+      have_css('#integrations.profile-settings-sidebar', { text: 'Integrations' })
     end
 
     before do