Add latest changes from gitlab-org/gitlab@master

changelogs/unreleased/bw-delete-jira-tracker-data-jobs.yml

@@ -0,0 +1,5 @@
+---
+title: Delete any outstanding BackfillJiraTrackerDeploymentType
+merge_request: 45219
+author:
+type: fixed

changelogs/unreleased/generic-packages-enable-by-default.yml

@@ -0,0 +1,5 @@
+---
+title: Add support for Generic packages
+merge_request: 45102
+author:
+type: added

config/feature_flags/development/generic_packages.yml

@@ -4,4 +4,4 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/40045
 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/239133
 group: group::release management
 type: development
-default_enabled: false
+default_enabled: true

db/migrate/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class DropBackfillJiraTrackerDeploymentTypeJobs < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+  DROPPED_JOB_CLASS = 'BackfillJiraTrackerDeploymentType'.freeze
+  QUEUE = 'background_migration'.freeze
+
+  def up
+    sidekiq_queues.each do |queue|
+      queue.each do |job|
+        next unless job.args.first == DROPPED_JOB_CLASS
+
+        job.delete
+      end
+    end
+  end
+
+  def down
+    # no-op
+  end
+
+  def sidekiq_queues
+    [Sidekiq::ScheduledSet.new, Sidekiq::RetrySet.new, Sidekiq::Queue.new(QUEUE)]
+  end
+end

db/schema_migrations/20201014205300

@@ -0,0 +1 @@
+05352623a59d56c1633317ba7dbaba7d75bb8e529a748a90512dcf3641b8d3cb

doc/api/lint.md

@@ -64,7 +64,8 @@ Example responses:
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/29568) in GitLab 13.5.
 
 The CI lint returns an expanded version of the configuration. The expansion does not
-work for CI configuration added with [`include: local`](../ci/yaml/README.md#includelocal).
+work for CI configuration added with [`include: local`](../ci/yaml/README.md#includelocal),
+or with [`extends:`](../ci/yaml/README.md#extends).
 
 Example contents of a `.gitlab-ci.yml` passed to the CI Lint API with
 `include_merged_yaml` set as true:
@@ -145,3 +146,92 @@ Example responses:
   "warnings": []
 }
 ```
+
+## Use jq to create and process YAML & JSON payloads
+
+To `POST` a YAML configuration to the CI Lint endpoint, it must be properly escaped and JSON encoded.
+You can use `jq` and `curl` to escape and upload YAML to the GitLab API.
+
+### Escape YAML for JSON encoding
+
+To escape quotes and encode your YAML in a format suitable for embedding within
+a JSON payload, you can use `jq`. For example, create a file named `example-gitlab-ci.yml`:
+
+```yaml
+.api_test:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE=="merge_request_event"'
+      changes:
+        - src/api/*
+deploy:
+  extends:
+    - .api_test
+  rules:
+    - when: manual
+      allow_failure: true
+  script:
+    - echo "hello world"
+```
+
+Next, use `jq` to escape and encode the YAML file into JSON:
+
+```shell
+jq --raw-input --slurp < example-gitlab-ci.yml
+```
+
+To escape and encode an input YAML file (`example-gitlab-ci.yml`), and `POST` it to the
+GitLab API using `curl` and `jq` in a one-line command:
+
+```shell
+jq --null-input --arg yaml "$(<example-gitlab-ci.yml)" '.content=$yaml' \
+| curl 'https://gitlab.com/api/v4/ci/lint?include_merged_yaml=true' \
+--header 'Content-Type: application/json' \
+--data @-
+```
+
+### Parse a CI Lint response
+
+To reformat the CI Lint response, you can use `jq`. You can pipe the CI Lint response to `jq`,
+or store the API response as a text file and provide it as an argument:
+
+```shell
+jq --raw-output '.merged_yaml | fromjson' <your_input_here>
+```
+
+Example input:
+
+```json
+{"status":"valid","errors":[],"merged_yaml":"---\n:.api_test:\n  :rules:\n  - :if: $CI_PIPELINE_SOURCE==\"merge_request_event\"\n    :changes:\n    - src/api/*\n:deploy:\n  :rules:\n  - :when: manual\n    :allow_failure: true\n  :extends:\n  - \".api_test\"\n  :script:\n  - echo \"hello world\"\n"}
+```
+
+Becomes:
+
+```yaml
+:.api_test:
+  :rules:
+  - :if: $CI_PIPELINE_SOURCE=="merge_request_event"
+    :changes:
+    - src/api/*
+:deploy:
+  :rules:
+  - :when: manual
+    :allow_failure: true
+  :extends:
+  - ".api_test"
+  :script:
+  - echo "hello world"
+```
+
+With a one-line command, you can:
+
+1. Escape the YAML
+1. Encode it in JSON
+1. POST it to the API with curl
+1. Format the response
+
+```shell
+jq --null-input --arg yaml "$(<example-gitlab-ci.yml)" '.content=$yaml' \
+| curl 'https://gitlab.com/api/v4/ci/lint?include_merged_yaml=true' \
+--header 'Content-Type: application/json' --data @- \
+| jq --raw-output '.merged_yaml | fromjson'
+```

doc/user/packages/generic_packages/index.md

@@ -7,11 +7,11 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 # GitLab Generic Packages Repository **(CORE)**
 
 > - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/4209) in GitLab 13.5.
-> - It's [deployed behind a feature flag](../../../user/feature_flags.md), disabled by default.
-> - It's disabled on GitLab.com.
+> - It's [deployed behind a feature flag](../../../user/feature_flags.md), enabled by default.
+> - It's enabled on GitLab.com.
 > - It's able to be enabled or disabled per-project.
-> - It's not recommended for production use.
-> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-generic-packages-in-the-package-registry).
+> - It's recommended for production use.
+> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-generic-packages-in-the-package-registry).
 
 CAUTION: **Warning:**
 This feature might not be available to you. Check the **version history** note above for details.
@@ -115,10 +115,10 @@ download:
 
 ### Enable or disable generic packages in the Package Registry
 
-Support for generic packages is under development and not ready for production use. It is
-deployed behind a feature flag that is **disabled by default**.
+Support for generic packages is under development but ready for production use.
+It is deployed behind a feature flag that is **enabled by default**.
 [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
-can enable it.
+can opt to disable it.
 
 To enable it:
 

lib/api/generic_packages.rb

@@ -101,7 +101,7 @@ module API
       include ::API::Helpers::Packages::BasicAuthHelpers
 
       def require_generic_packages_available!
-        not_found! unless Feature.enabled?(:generic_packages, project)
+        not_found! unless Feature.enabled?(:generic_packages, project, default_enabled: true)
       end
 
       def project

spec/lib/gitlab/graphql/authorize/authorize_field_service_spec.rb

@@ -19,24 +19,29 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
       options.reverse_merge!(null: true)
       field :test_field, field_type,
             authorize: field_authorizations,
-            resolve: -> (_, _, _) { resolved_value },
             **options
+
+      define_method :test_field do
+        resolved_value
+      end
     end
   end
 
-  let(:current_user) { double(:current_user) }
-
   subject(:service) { described_class.new(field) }
 
   describe '#authorized_resolve' do
-    let(:presented_object) { double('presented object') }
-    let(:presented_type) { double('parent type', object: presented_object) }
-    let(:query_type) { GraphQL::ObjectType.new }
-    let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)}
-    let(:query_context) { OpenStruct.new(schema: schema) }
-    let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema, context: query_context), values: { current_user: current_user }, object: nil) }
+    let_it_be(:current_user) { build(:user) }
+    let_it_be(:presented_object) { 'presented object' }
+    let_it_be(:query_type) { GraphQL::ObjectType.new }
+    let_it_be(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)}
+    let_it_be(:query) { GraphQL::Query.new(schema, document: nil, context: {}, variables: {}) }
+    let_it_be(:context) { GraphQL::Query::Context.new(query: query, values: { current_user: current_user }, object: nil) }
 
-    subject(:resolved) { service.authorized_resolve.call(presented_type, {}, context) }
+    let(:type_class) { type_with_field(custom_type, :read_field, presented_object) }
+    let(:type_instance) { type_class.authorized_new(presented_object, context) }
+    let(:field) { type_class.fields['testField'].to_graphql }
+
+    subject(:resolved) { service.authorized_resolve.call(type_instance, {}, context) }
 
     context 'scalar types' do
       shared_examples 'checking permissions on the presented object' do
@@ -48,7 +53,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
           expect(resolved).to eq('Resolved value')
         end
 
-        it "returns nil if the value wasn't authorized" do
+        it 'returns nil if the value was not authorized' do
           allow(Ability).to receive(:allowed?).and_return false
 
           expect(resolved).to be_nil
@@ -56,28 +61,28 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
       end
 
       context 'when the field is a built-in scalar type' do
-        let(:field) { type_with_field(GraphQL::STRING_TYPE, :read_field).fields['testField'].to_graphql }
+        let(:type_class) { type_with_field(GraphQL::STRING_TYPE, :read_field) }
         let(:expected_permissions) { [:read_field] }
 
         it_behaves_like 'checking permissions on the presented object'
       end
 
       context 'when the field is a list of scalar types' do
-        let(:field) { type_with_field([GraphQL::STRING_TYPE], :read_field).fields['testField'].to_graphql }
+        let(:type_class) { type_with_field([GraphQL::STRING_TYPE], :read_field) }
         let(:expected_permissions) { [:read_field] }
 
         it_behaves_like 'checking permissions on the presented object'
       end
 
       context 'when the field is sub-classed scalar type' do
-        let(:field) { type_with_field(Types::TimeType, :read_field).fields['testField'].to_graphql }
+        let(:type_class) { type_with_field(Types::TimeType, :read_field) }
         let(:expected_permissions) { [:read_field] }
 
         it_behaves_like 'checking permissions on the presented object'
       end
 
       context 'when the field is a list of sub-classed scalar types' do
-        let(:field) { type_with_field([Types::TimeType], :read_field).fields['testField'].to_graphql }
+        let(:type_class) { type_with_field([Types::TimeType], :read_field) }
         let(:expected_permissions) { [:read_field] }
 
         it_behaves_like 'checking permissions on the presented object'
@@ -86,7 +91,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
 
     context 'when the field is a connection' do
       context 'when it resolves to nil' do
-        let(:field) { type_with_field(Types::QueryType.connection_type, :read_field, nil).fields['testField'].to_graphql }
+        let(:type_class) { type_with_field(Types::QueryType.connection_type, :read_field, nil) }
 
         it 'does not fail when authorizing' do
           expect(resolved).to be_nil
@@ -97,7 +102,11 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
     context 'when the field is a specific type' do
       let(:custom_type) { type(:read_type) }
       let(:object_in_field) { double('presented in field') }
-      let(:field) { type_with_field(custom_type, :read_field, object_in_field).fields['testField'].to_graphql }
+
+      let(:type_class) { type_with_field(custom_type, :read_field, object_in_field) }
+      let(:type_instance) { type_class.authorized_new(object_in_field, context) }
+
+      subject(:resolved) { service.authorized_resolve.call(type_instance, {}, context) }
 
       it 'checks both field & type permissions' do
         spy_ability_check_for(:read_field, object_in_field, passed: true)
@@ -114,7 +123,7 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
       end
 
       context 'when the field is not nullable' do
-        let(:field) { type_with_field(custom_type, [], object_in_field, null: false).fields['testField'].to_graphql }
+        let(:type_class) { type_with_field(custom_type, :read_field, object_in_field, null: false) }
 
         it 'returns nil when viewing is not allowed' do
           spy_ability_check_for(:read_type, object_in_field, passed: false)
@@ -127,7 +136,9 @@ RSpec.describe Gitlab::Graphql::Authorize::AuthorizeFieldService do
         let(:object_1) { double('presented in field 1') }
         let(:object_2) { double('presented in field 2') }
         let(:presented_types) { [double(object: object_1), double(object: object_2)] }
-        let(:field) { type_with_field([custom_type], :read_field, presented_types).fields['testField'].to_graphql }
+
+        let(:type_class) { type_with_field([custom_type], :read_field, presented_types) }
+        let(:type_instance) { type_class.authorized_new(presented_types, context) }
 
         it 'checks all permissions' do
           allow(Ability).to receive(:allowed?) { true }

spec/migrations/20201014205300_drop_backfill_jira_tracker_deployment_type_jobs_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20201014205300_drop_backfill_jira_tracker_deployment_type_jobs.rb')
+
+RSpec.describe DropBackfillJiraTrackerDeploymentTypeJobs, :sidekiq, :redis, schema: 2020_10_14_205300 do
+  subject(:migration) { described_class.new }
+
+  describe '#up' do
+    let(:retry_set) { Sidekiq::RetrySet.new }
+    let(:scheduled_set) { Sidekiq::ScheduledSet.new }
+
+    context 'there are only affected jobs on the queue' do
+      let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1] } }
+      let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) }
+
+      it 'removes enqueued BackfillJiraTrackerDeploymentType background jobs' do
+        Sidekiq::Testing.disable! do # https://github.com/mperham/sidekiq/wiki/testing#api Sidekiq's API does not have a testing mode
+          retry_set.schedule(1.hour.from_now, payload)
+          scheduled_set.schedule(1.hour.from_now, payload)
+          Sidekiq::Client.push(queue_payload)
+
+          expect { migration.up }.to change { Sidekiq::Queue.new(described_class::QUEUE).size }.from(1).to(0)
+          expect(retry_set.size).to eq(0)
+          expect(scheduled_set.size).to eq(0)
+        end
+      end
+    end
+
+    context 'there are not any affected jobs on the queue' do
+      let(:payload) { { 'class' => ::BackgroundMigrationWorker, 'args' => ['SomeOtherClass', 1] } }
+      let(:queue_payload) { payload.merge('queue' => described_class::QUEUE) }
+
+      it 'skips other enqueued jobs' do
+        Sidekiq::Testing.disable! do
+          retry_set.schedule(1.hour.from_now, payload)
+          scheduled_set.schedule(1.hour.from_now, payload)
+          Sidekiq::Client.push(queue_payload)
+
+          expect { migration.up }.not_to change { Sidekiq::Queue.new(described_class::QUEUE).size }
+          expect(retry_set.size).to eq(1)
+          expect(scheduled_set.size).to eq(1)
+        end
+      end
+    end
+
+    context 'other queues' do
+      it 'does not modify them' do
+        Sidekiq::Testing.disable! do
+          Sidekiq::Client.push('queue' => 'other', 'class' => ::BackgroundMigrationWorker, 'args' => ['SomeOtherClass', 1])
+          Sidekiq::Client.push('queue' => 'other', 'class' => ::BackgroundMigrationWorker, 'args' => [described_class::DROPPED_JOB_CLASS, 1])
+
+          expect { migration.up }.not_to change { Sidekiq::Queue.new('other').size }
+        end
+      end
+    end
+  end
+end