Add latest changes from gitlab-org/gitlab@master

GITALY_SERVER_VERSION

@@ -1 +1 @@
-57048c3d003ebf72ba8342a03b2f6d510193e49e
+36aaf4e475fdcc4ae89f14772662fa89125d7716

app/models/ci/secure_file.rb

@@ -11,10 +11,13 @@ module Ci
     self.limit_scope = :project
     self.limit_name = 'project_ci_secure_files'
 
+    attr_accessor :file_checksum
+
     belongs_to :project, optional: false
 
     validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT }
     validates :checksum, :file_store, :name, :permissions, :project_id, presence: true
+    validate :validate_upload_checksum, on: :create
 
     before_validation :assign_checksum
 
@@ -33,5 +36,11 @@ module Ci
     def assign_checksum
       self.checksum = file.checksum if file.present? && file_changed?
     end
+
+    def validate_upload_checksum
+      unless self.file_checksum.nil?
+        errors.add(:file_checksum, _("Secure Files|File did not match the provided checksum")) unless self.file_checksum == self.checksum
+      end
+    end
   end
 end

doc/administration/packages/index.md

@@ -10,9 +10,7 @@ GitLab Packages allows organizations to use GitLab as a private repository
 for a variety of common package managers. Users are able to build and publish
 packages, which can be easily consumed as a dependency in downstream projects.
 
-The Packages feature allows GitLab to act as a repository for the following:
-
-The Package Registry supports the following formats:
+The Packages feature allows GitLab to act as a repository and supports the following formats:
 
 | Package type                                                      | GitLab version |
 |-------------------------------------------------------------------|----------------|

doc/api/secure_files.md

@@ -101,12 +101,13 @@ POST /projects/:project_id/secure_files
 
 Supported attributes:
 
-| Attribute     | Type           | Required               | Description |
-|---------------|----------------|------------------------|-------------|
-| `project_id`  | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. |
-| `name`        | string         | **{check-circle}** Yes | The `name` of the file being uploaded. |
-| `file`        | file           | **{check-circle}** Yes | The `file` being uploaded. |
-| `permissions` | string         | **{dotted-circle}** No | The file is created with the specified permissions when created in the CI/CD job. Available types are: `read_only` (default), `read_write`, and `execute`. |
+| Attribute       | Type           | Required               | Description |
+|-----------------|----------------|------------------------|-------------|
+| `project_id`    | integer/string | **{check-circle}** Yes | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user. |
+| `name`          | string         | **{check-circle}** Yes | The `name` of the file being uploaded. |
+| `file`          | file           | **{check-circle}** Yes | The `file` being uploaded. |
+| `file_checksum` | file           | **{dotted-circle}** No | An optional sha256 checksum of the file to be uploaded. If provided, the checksum must match the uploaded file, or the upload will fail to validate. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/355653) in GitLab 14.10. |
+| `permissions`   | string         | **{dotted-circle}** No | The file is created with the specified permissions when created in the CI/CD job. Available types are: `read_only` (default), `read_write`, and `execute`. |
 
 Example request:
 

doc/tutorials/how_git_works.md

@@ -0,0 +1,267 @@
+---
+stage: none
+group: unassigned
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+---
+
+# Make your first Git commit
+
+This tutorial is going to teach you a little bit about how Git works, and walk
+you through the steps of creating your own project and editing a file.
+
+When you're done, you'll have a project where you can practice using Git.
+
+## What you need
+
+Before you begin:
+
+- [Install Git on your local machine](../topics/git/how_to_install_git/index.md).
+- Ensure you can sign in to an instance of GitLab. If your organization doesn't
+  have GitLab, create an account on GitLab.com.
+- [Create SSH keys and add them to GitLab](../ssh/index.md). SSH keys are how you
+  securely communicate between your computer and GitLab.
+
+## What is Git?
+
+Before we jump into steps, let's go over some basic Git concepts.
+
+Git is a version control system. It's used to track changes to files.
+
+You store files, like code or documents, in a Git *repository*. When you want to edit the files, you
+*clone* the repository to your computer, make the changes, and *push* your changes
+back to the repository. In GitLab, a Git repository is located in
+a *project*.
+
+Each time you push a change, Git records it as a unique *commit*. These commits make up
+the history of when and how a file changed, and who changed it.
+
+```mermaid
+graph LR
+    subgraph Repository commit history
+    A(Author: Alex<br>Date: 3 Jan at 1PM<br>Commit message: Added sales figures for January<br> Commit ID: 123abc12) --->  B
+    B(Author: Sam<br>Date: 4 Jan at 10AM<br>Commit message: Removed outdated marketing information<br> Commit ID: aabb1122) ---> C
+    C(Author: Zhang<br>Date: 5 Jan at 3PM<br>Commit message: Added a new 'Invoices' file<br> Commit ID: ddee4455)
+    end
+```
+
+When you work in a Git repository, you work in *branches*. By default, the contents
+of a repository are in a default branch. To make changes, you:
+
+1. Create your own branch, which is a snapshot of the default branch at the time
+   you create it.
+1. Make changes and push them to your branch. Each push creates a commit.
+1. When you're ready, *merge* your branch into the default branch.
+
+```mermaid
+flowchart LR
+    subgraph Default branch
+    A[Commit] --> B[Commit] --> C[Commit] --> D[Commit]
+    end
+    subgraph My branch
+    B --1. Create my branch--> E(Commit)
+    E --2. Add my commit--> F(Commit)
+    F --2. Add my commit--> G(Commit)
+    G --3. Merge my branch to default--> D
+    end
+```
+
+If this all feels a bit overwhelming, hang in there. You're about to see these concepts in action.
+
+## Steps
+
+Here's an overview of what we're going to do:
+
+1. [Create a sample project](#create-a-sample-project).
+1. [Clone the repository](#clone-the-repository).
+1. [Create a branch and make your changes](#create-a-branch-and-make-changes).
+1. [Commit and push your changes](#commit-and-push-your-changes).
+1. [Merge your changes](#merge-your-changes).
+1. [View your changes in GitLab](#view-your-changes-in-gitlab).
+
+### Create a sample project
+
+To start, create a sample project in GitLab.
+
+1. In GitLab, on the top bar, select **Menu > Projects > Create new project**.
+1. Select **Create blank project**.
+1. For **Project name**, enter `My sample project`. The project slug is generated for you.
+   This slug is the URL you can use to access the project after it's created.
+1. Ensure **Initialize repository with a README** is selected.
+   How you complete the other fields is up to you.
+1. Select **Create project**.
+
+### Clone the repository
+
+Now you can clone the repository in your project. *Cloning* a repository means you're creating
+a copy on your computer, or wherever you want to store and work with the files.
+
+1. On your project page, select **Clone**. Copy the URL for **Clone with SSH**.
+
+   ![Clone a project with SSH](img/clone_project_v14_9.png)
+
+1. Open a terminal on your computer and go to the directory
+   where you want to clone the files.
+
+1. Enter `git clone` and paste the URL:
+
+   ```shell
+   git clone git@gitlab.com:gitlab-example/my-sample-project.git
+   ```
+
+1. Go to the directory:
+
+   ```shell
+   cd my-sample-project
+   ```
+
+1. By default, you've cloned the default branch for the repository. Usually this
+   branch is `main`. To be sure, get the name of the default branch:
+
+   ```shell
+   git branch
+   ```
+
+   The branch you're on is marked with an asterisk.
+   Press `Q` on your keyboard to return to the main terminal
+   window.
+
+### Create a branch and make changes
+
+Now that you have a copy of the repository, create your own branch so you can
+work on your changes independently.
+
+1. Create a new branch called `example-tutorial-branch`.
+
+   ```shell
+   git checkout -b example-tutorial-branch
+   ```
+
+1. In a text editor like Visual Studio Code, Sublime, `vi`, or any other editor,
+   open the README.md file and add this text:
+
+   ```plaintext
+   Hello world! I'm using Git!
+   ```
+
+1. Save the file.
+
+1. Git keeps track of changed files. To confirm which files have changed, get
+   the status.
+
+   ```shell
+   git status
+   ```
+
+   You should get output similar to the following:
+
+   ```shell
+   On branch example-tutorial-branch
+   Changes not staged for commit:
+   (use "git add <file>..." to update what will be committed)
+   (use "git restore <file>..." to discard changes in working directory)
+   modified:   README.md
+
+   no changes added to commit (use "git add" and/or "git commit -a")
+   ```
+
+### Commit and push your changes
+
+You've made changes to a file in your repository. Now it's time to record
+those changes by making your first commit.
+
+1. Add the `README.md` file to the *staging* area. The staging area is where you
+   put files before you commit them.
+
+   ```shell
+   git add README.md
+   ```
+
+1. Confirm the file is staged:
+
+   ```shell
+   git status
+   ```
+
+   You should get output similar to the following, and the filename should be in
+   green text.
+
+   ```shell
+   On branch example-tutorial-branch
+   Changes to be committed:
+   (use "git restore --staged <file>..." to unstage)
+   modified:   README.md
+   ```
+
+1. Now commit the staged file, and include a message
+   that describes the change you made. Make sure you surround the message in double
+   quotes (").
+
+   ```shell
+   git commit -m "I added text to the README file"
+   ```
+
+1. The change has been committed to your branch, but your branch and its commits
+   are still only available on your computer. No one else has access to them yet.
+   Push your branch to GitLab:
+
+   ```shell
+   git push origin example-tutorial-branch
+   ```
+
+Your branch is now available on GitLab and visible to other users in your project.
+
+![Branches dropdown list](img/branches_dropdown_v14_10.png)
+
+### Merge your changes
+
+Now you're ready to merge the changes from your `example-tutorial-branch` branch
+to the default branch (`main`).
+
+1. Check out the default branch for your repository.
+
+   ```shell
+   git checkout main
+   ```
+
+1. Merge your branch into the default branch.
+
+   ```shell
+   git merge example-tutorial-branch
+   ```
+
+1. Push the changes.
+
+   ```shell
+   git push
+   ```
+
+NOTE:
+For this tutorial, you merge your branch directly to the default branch for your
+repository. In GitLab, you typically use a [merge request](../user/project/merge_requests/)
+to merge your branch.
+
+### View your changes in GitLab
+
+You did it! You updated the `README.md` file in your branch, and you merged those changes
+into the `main` branch.
+
+Let's look in the UI and confirm it.
+
+1. In your project, scroll down and view the `README.md` file.
+   Your changes should be visible.
+1. Above the list of files, select **History**.
+   Your commit and commit message should display.
+
+Now you can change back to your personal branch (`git checkout example-tutorial-branch`)
+and continue updating or even creating files. Type `git status` to view the status
+of your changes and commit with abandon.
+
+Don't worry if you mess things up. Everything in Git can be reverted, and if you
+find you can't recover, you can always create a new branch and start again.
+
+Nice work.
+
+## Find more Git learning resources
+
+- Get a complete introduction to Git in the <i class="fa fa-youtube-play youtube" aria-hidden="true"></i> [Git for GitLab](https://www.youtube.com/watch?v=4lxvVj7wlZw) beginner's course (1h 33m).
+- Find other tutorials about Git and GitLab on the [tutorials page](index.md).

lib/api/ci/secure_files.rb

@@ -62,12 +62,14 @@ module API
             requires :name, type: String, desc: 'The name of the file'
             requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The secure file to be uploaded'
             optional :permissions, type: String, desc: 'The file permissions', default: 'read_only', values: %w[read_only read_write execute]
+            optional :file_checksum, type: String, desc: 'An optional sha256 checksum of the file to be uploaded'
           end
           route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: true
           post ':id/secure_files' do
             secure_file = user_project.secure_files.new(
               name: params[:name],
-              permissions: params[:permissions] || :read_only
+              permissions: params[:permissions] || :read_only,
+              file_checksum: params[:file_checksum]
             )
 
             secure_file.file = params[:file]

lib/gitlab/usage_data_queries.rb

@@ -50,7 +50,7 @@ module Gitlab
 
       def alt_usage_data(value = nil, fallback: FALLBACK, &block)
         if block_given?
-          { alt_usage_data_block: "non-SQL usage data block" }
+          { alt_usage_data_block: block.to_s }
         else
           { alt_usage_data_value: value }
         end
@@ -58,9 +58,9 @@ module Gitlab
 
       def redis_usage_data(counter = nil, &block)
         if block_given?
-          { redis_usage_data_block: "non-SQL usage data block" }
+          { redis_usage_data_block: block.to_s }
         elsif counter.present?
-          { redis_usage_data_counter: counter.to_s }
+          { redis_usage_data_counter: counter }
         end
       end
 
@@ -74,15 +74,6 @@ module Gitlab
       def epics_deepest_relationship_level
         { epics_deepest_relationship_level: 0 }
       end
-
-      # Do not include Devise omniauth providers for consistency between local and CI-run specs
-      def auth_providers
-        AuthenticationEvent::STATIC_PROVIDERS
-      end
-
-      def omniauth_provider_names
-        []
-      end
     end
   end
 end

lib/tasks/gitlab/usage_data.rake

@@ -51,17 +51,6 @@ namespace :gitlab do
       File.write(Gitlab::UsageDataCounters::CiTemplateUniqueCounter::KNOWN_EVENTS_FILE_PATH, banner + YAML.dump(all_includes).gsub(/ *$/m, ''))
     end
 
-    desc 'GitLab | UsageDataMetrics | Generate raw SQL metrics queries fixture for RSpec'
-    task generate_sql_metrics_fixture: :environment do
-      path = Rails.root.join('spec/fixtures/lib/gitlab/usage/sql_metrics_queries.json')
-
-      queries = Timecop.freeze(2021, 1, 1) do
-        Gitlab::Usage::ServicePingReport.for(output: :metrics_queries)
-      end
-
-      File.write(path, Gitlab::Json.pretty_generate(queries))
-    end
-
     def ci_template_includes_hash(source, template_directory = nil)
       Gitlab::UsageDataCounters::CiTemplateUniqueCounter.ci_templates("lib/gitlab/ci/templates/#{template_directory}").map do |template|
         expanded_template_name = Gitlab::UsageDataCounters::CiTemplateUniqueCounter.expand_template_name("#{template_directory}/#{template}")

locale/gitlab.pot

@@ -32856,6 +32856,9 @@ msgstr ""
 msgid "Secure Files"
 msgstr ""
 
+msgid "Secure Files|File did not match the provided checksum"
+msgstr ""
+
 msgid "Secure token that identifies an external storage request."
 msgstr ""
 

spec/lib/gitlab/usage_data_queries_spec.rb

@@ -34,14 +34,14 @@ RSpec.describe Gitlab::UsageDataQueries do
   describe '.redis_usage_data' do
     subject(:redis_usage_data) { described_class.redis_usage_data { 42 } }
 
-    it 'returns a stringified class for redis_usage_data with a counter call' do
+    it 'returns a class for redis_usage_data with a counter call' do
       expect(described_class.redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter))
-        .to eq(redis_usage_data_counter: "Gitlab::UsageDataCounters::WikiPageCounter")
+        .to eq(redis_usage_data_counter: Gitlab::UsageDataCounters::WikiPageCounter)
     end
 
-    it 'returns a placeholder string for redis_usage_data with a block' do
+    it 'returns a stringified block for redis_usage_data with a block' do
       is_expected.to include(:redis_usage_data_block)
-      expect(redis_usage_data[:redis_usage_data_block]).to eq('non-SQL usage data block')
+      expect(redis_usage_data[:redis_usage_data_block]).to start_with('#<Proc:')
     end
   end
 
@@ -53,8 +53,8 @@ RSpec.describe Gitlab::UsageDataQueries do
         .to eq(alt_usage_data_value: 1)
     end
 
-    it 'returns a placeholder string for alt_usage_data with a block' do
-      expect(alt_usage_data[:alt_usage_data_block]).to eq('non-SQL usage data block')
+    it 'returns a stringified block for alt_usage_data with a block' do
+      expect(alt_usage_data[:alt_usage_data_block]).to start_with('#<Proc:')
     end
   end
 

spec/requests/api/ci/secure_files_spec.rb

@@ -232,6 +232,22 @@ RSpec.describe API::Ci::SecureFiles do
         expect(Base64.encode64(response.body)).to eq(Base64.encode64(fixture_file_upload('spec/fixtures/ci_secure_files/upload-keystore.jks').read))
       end
 
+      it 'uploads and validates a secure file with a provided checksum' do
+        params = {
+          file: fixture_file_upload('spec/fixtures/ci_secure_files/upload-keystore.jks'),
+          name: 'upload-keystore.jks',
+          permissions: 'execute',
+          file_checksum: Digest::SHA256.hexdigest(File.read(fixture_file_upload('spec/fixtures/ci_secure_files/upload-keystore.jks')))
+        }
+
+        expect do
+          post api("/projects/#{project.id}/secure_files", maintainer), params: params
+        end.to change {project.secure_files.count}.by(1)
+
+        expect(response).to have_gitlab_http_status(:created)
+        expect(json_response['name']).to eq('upload-keystore.jks')
+      end
+
       it 'returns an error when the file checksum fails to validate' do
         secure_file.update!(checksum: 'foo')
 
@@ -242,6 +258,22 @@ RSpec.describe API::Ci::SecureFiles do
         expect(response.code).to eq("500")
       end
 
+      it 'returns an error when the user provided file checksum fails to validate' do
+        post_params = {
+          file: fixture_file_upload('spec/fixtures/ci_secure_files/upload-keystore.jks'),
+          name: 'upload-keystore.jks',
+          permissions: 'read_write',
+          file_checksum: 'foo'
+        }
+
+        expect do
+          post api("/projects/#{project.id}/secure_files", maintainer), params: post_params
+        end.not_to change { project.secure_files.count }
+
+        expect(response).to have_gitlab_http_status(:bad_request)
+        expect(json_response['message']['file_checksum']).to include(_("Secure Files|File did not match the provided checksum"))
+      end
+
       it 'returns an error when no file is uploaded' do
         post_params = {
           name: 'upload-keystore.jks'