diff --git a/app/assets/javascripts/profile/preferences/components/profile_preferences.vue b/app/assets/javascripts/profile/preferences/components/profile_preferences.vue index a0129dd536b..757a66ef148 100644 --- a/app/assets/javascripts/profile/preferences/components/profile_preferences.vue +++ b/app/assets/javascripts/profile/preferences/components/profile_preferences.vue @@ -127,20 +127,18 @@ export default { :config="$options.integrationViewConfigs[view.name]" /> -
-
-
- - {{ $options.i18n.saveChanges }} - -
+
+
+ + {{ $options.i18n.saveChanges }} +
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7758620f605..1ca291a659b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -53,6 +53,7 @@ module Ci end has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build + has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build accepts_nested_attributes_for :runner_session, update_only: true accepts_nested_attributes_for :job_variables diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb new file mode 100644 index 00000000000..05bdb3d8b7b --- /dev/null +++ b/app/models/ci/build_trace_metadata.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Ci + class BuildTraceMetadata < Ci::ApplicationRecord + self.table_name = 'ci_build_trace_metadata' + self.primary_key = :build_id + + belongs_to :build, class_name: 'Ci::Build' + belongs_to :trace_artifact, class_name: 'Ci::JobArtifact' + + validates :build, presence: true + end +end diff --git a/app/models/user_interacted_project.rb b/app/models/user_interacted_project.rb index 4c8cc5fc83a..1c7515894fe 100644 --- a/app/models/user_interacted_project.rb +++ b/app/models/user_interacted_project.rb @@ -24,16 +24,8 @@ class UserInteractedProject < ApplicationRecord } cached_exists?(**attributes) do - transaction(requires_new: true) do - where(attributes).select(1).first || create!(attributes) - true # not caching the whole record here for now - rescue ActiveRecord::RecordNotUnique - # Note, above queries are not atomic and prone - # to race conditions (similar like #find_or_create!). - # In the case where we hit this, the record we want - # already exists - shortcut and return. - true - end + where(attributes).exists? || UserInteractedProject.insert_all([attributes], unique_by: %w(project_id user_id)) + true end end diff --git a/app/views/profiles/show.html.haml b/app/views/profiles/show.html.haml index a314d5866ff..6eba0309a4f 100644 --- a/app/views/profiles/show.html.haml +++ b/app/views/profiles/show.html.haml @@ -122,10 +122,9 @@ = f.check_box :include_private_contributions, label: s_('Profiles|Include private contributions on my profile'), wrapper_class: 'mb-2', inline: true .help-block = s_("Profiles|Choose to show contributions of private projects on your public profile without any project, repository or organization information") - .row.gl-justify-content-end.gl-mt-5 - .col-lg-8.gl-display-flex - = f.submit s_("Profiles|Update profile settings"), class: 'gl-button btn btn-confirm gl-mr-3' - = link_to _("Cancel"), user_path(current_user), class: 'gl-button btn btn-default btn-cancel' + %hr + = f.submit s_("Profiles|Update profile settings"), class: 'gl-button btn btn-confirm gl-mr-3' + = link_to _("Cancel"), user_path(current_user), class: 'gl-button btn btn-default btn-cancel' .modal.modal-profile-crop{ data: { cropper_css_path: ActionController::Base.helpers.stylesheet_path('lazy_bundles/cropper.css') } } .modal-dialog diff --git a/db/migrate/20210813111909_create_ci_build_trace_metadata.rb b/db/migrate/20210813111909_create_ci_build_trace_metadata.rb new file mode 100644 index 00000000000..d8b7fd656e0 --- /dev/null +++ b/db/migrate/20210813111909_create_ci_build_trace_metadata.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class CreateCiBuildTraceMetadata < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + def up + with_lock_retries do + create_table :ci_build_trace_metadata, id: false, if_not_exists: true do |t| + t.references :build, + index: false, + primary_key: true, + default: nil, + foreign_key: { to_table: :ci_builds, on_delete: :cascade }, + type: :bigint, + null: false + + t.bigint :trace_artifact_id + t.integer :archival_attempts, default: 0, null: false, limit: 2 + t.binary :checksum + t.binary :remote_checksum + + t.index :trace_artifact_id + end + end + end + + def down + with_lock_retries do + drop_table :ci_build_trace_metadata, if_exists: true + end + end +end diff --git a/db/migrate/20210817084338_add_foreign_key_from_ci_build_metadata_to_ci_job_artifacts.rb b/db/migrate/20210817084338_add_foreign_key_from_ci_build_metadata_to_ci_job_artifacts.rb new file mode 100644 index 00000000000..032b15234d8 --- /dev/null +++ b/db/migrate/20210817084338_add_foreign_key_from_ci_build_metadata_to_ci_job_artifacts.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddForeignKeyFromCiBuildMetadataToCiJobArtifacts < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :ci_build_trace_metadata, + :ci_job_artifacts, + column: :trace_artifact_id, + on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :ci_build_trace_metadata, column: :trace_artifact_id + end + end +end diff --git a/db/schema_migrations/20210813111909 b/db/schema_migrations/20210813111909 new file mode 100644 index 00000000000..c6073d273da --- /dev/null +++ b/db/schema_migrations/20210813111909 @@ -0,0 +1 @@ +78ea79c2157acb8a0b29afb4d2f3de6965fb0ea885b5d5f939f22ccda1d53b1e \ No newline at end of file diff --git a/db/schema_migrations/20210817084338 b/db/schema_migrations/20210817084338 new file mode 100644 index 00000000000..ca7219b7ea6 --- /dev/null +++ b/db/schema_migrations/20210817084338 @@ -0,0 +1 @@ +e301e1be82c96e62bba0f958c54dda4633b28553246d4c576d6de33cd33e0a50 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 91c02925a4e..22e89176ce3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10575,6 +10575,14 @@ CREATE SEQUENCE ci_build_trace_chunks_id_seq ALTER SEQUENCE ci_build_trace_chunks_id_seq OWNED BY ci_build_trace_chunks.id; +CREATE TABLE ci_build_trace_metadata ( + build_id bigint NOT NULL, + trace_artifact_id bigint, + archival_attempts smallint DEFAULT 0 NOT NULL, + checksum bytea, + remote_checksum bytea +); + CREATE TABLE ci_builds ( id integer NOT NULL, status character varying, @@ -21364,6 +21372,9 @@ ALTER TABLE ONLY ci_build_report_results ALTER TABLE ONLY ci_build_trace_chunks ADD CONSTRAINT ci_build_trace_chunks_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_build_trace_metadata + ADD CONSTRAINT ci_build_trace_metadata_pkey PRIMARY KEY (build_id); + ALTER TABLE ONLY dep_ci_build_trace_sections ADD CONSTRAINT ci_build_trace_sections_pkey PRIMARY KEY (build_id, section_name_id); @@ -23322,6 +23333,8 @@ CREATE INDEX index_ci_build_report_results_on_project_id ON ci_build_report_resu CREATE UNIQUE INDEX index_ci_build_trace_chunks_on_build_id_and_chunk_index ON ci_build_trace_chunks USING btree (build_id, chunk_index); +CREATE INDEX index_ci_build_trace_metadata_on_trace_artifact_id ON ci_build_trace_metadata USING btree (trace_artifact_id); + CREATE UNIQUE INDEX index_ci_builds_metadata_on_build_id ON ci_builds_metadata USING btree (build_id); CREATE INDEX index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts ON ci_builds_metadata USING btree (build_id) WHERE (has_exposed_artifacts IS TRUE); @@ -26200,6 +26213,9 @@ ALTER TABLE ONLY epics ALTER TABLE ONLY geo_container_repository_updated_events ADD CONSTRAINT fk_212c89c706 FOREIGN KEY (container_repository_id) REFERENCES container_repositories(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_build_trace_metadata + ADD CONSTRAINT fk_21d25cac1a FOREIGN KEY (trace_artifact_id) REFERENCES ci_job_artifacts(id) ON DELETE CASCADE; + ALTER TABLE ONLY users_star_projects ADD CONSTRAINT fk_22cd27ddfc FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -28027,6 +28043,9 @@ ALTER TABLE ONLY analytics_cycle_analytics_group_stages ALTER TABLE ONLY metrics_dashboard_annotations ADD CONSTRAINT fk_rails_aeb11a7643 FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_build_trace_metadata + ADD CONSTRAINT fk_rails_aebc78111f FOREIGN KEY (build_id) REFERENCES ci_builds(id) ON DELETE CASCADE; + ALTER TABLE ONLY bulk_import_trackers ADD CONSTRAINT fk_rails_aed566d3f3 FOREIGN KEY (bulk_import_entity_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE; diff --git a/doc/ci/enable_or_disable_ci.md b/doc/ci/enable_or_disable_ci.md index bc30cf7c799..421bca9e324 100644 --- a/doc/ci/enable_or_disable_ci.md +++ b/doc/ci/enable_or_disable_ci.md @@ -41,7 +41,7 @@ To enable or disable GitLab CI/CD pipelines in your project: 1. On the top bar, select **Menu > Projects** and find your project. 1. On the left sidebar, select **Settings > General**. 1. Expand **Visibility, project features, permissions**. -1. In the **Repository** section, turn on or off **Pipelines** as required. +1. In the **Repository** section, turn on or off **CI/CD** as required. **Project visibility** also affects pipeline visibility. If set to: diff --git a/doc/development/cicd/templates.md b/doc/development/cicd/templates.md index 204287d7b59..03823a4b712 100644 --- a/doc/development/cicd/templates.md +++ b/doc/development/cicd/templates.md @@ -386,6 +386,6 @@ If you're unsure if it's secure or not, you need to ask security experts for cro After your CI/CD template MR is created and labeled with `ci::templates`, DangerBot suggests one reviewer and one maintainer that can review your code. When your merge -request is ready for review, please `@mention` the reviewer and ask them to review -your CI/CD template changes. See details in the merge request that added +request is ready for review, please [mention](../../user/project/issues/issue_data_and_actions.md#mentions) +the reviewer and ask them to review your CI/CD template changes. See details in the merge request that added [a DangerBot task for CI/CD template MRs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/44688). diff --git a/doc/development/contributing/index.md b/doc/development/contributing/index.md index fc9ed00d530..fe7dc52d077 100644 --- a/doc/development/contributing/index.md +++ b/doc/development/contributing/index.md @@ -129,8 +129,8 @@ The general flow of contributing to GitLab is: 1. In the merge request's description: - Ensure you provide complete and accurate information. - Review the provided checklist. -1. Assign the merge request (if possible) to, or `@mention`, one of the - [code owners](../../user/project/code_owners.md) for the relevant project, +1. Assign the merge request (if possible) to, or [mention](../../user/project/issues/issue_data_and_actions.md#mentions), + one of the [code owners](../../user/project/code_owners.md) for the relevant project, and explain that you are ready for review. When you submit code to GitLab, we really want it to get merged! However, we always review @@ -167,18 +167,18 @@ GitLab will do its best to review community contributions as quickly as possible appointed developers review community contributions daily. Look at the [team page](https://about.gitlab.com/company/team/) for the merge request coach who specializes in the type of code you have written and mention them in the merge request. For example, if you have -written some front-end code, you should `@mention` the frontend merge request coach. If -your code has multiple disciplines, you may `@mention` multiple merge request coaches. +written some front-end code, you should mention the frontend merge request coach. If +your code has multiple disciplines, you may mention multiple merge request coaches. GitLab receives a lot of community contributions. If your code has not been reviewed within two -working days of its initial submission, feel free to `@mention` all merge request coaches with +working days of its initial submission, feel free to mention all merge request coaches with `@gitlab-org/coaches` to get their attention. When submitting code to GitLab, you may feel that your contribution requires the aid of an external library. If your code includes an external library, please provide a link to the library, as well as reasons for including it. -`@mention` a maintainer in merge requests that contain: +Mention a maintainer in merge requests that contain: - More than 500 changes. - Any major [breaking changes](#breaking-changes). diff --git a/doc/development/documentation/styleguide/word_list.md b/doc/development/documentation/styleguide/word_list.md index ab70c5f4965..9e921bb30f0 100644 --- a/doc/development/documentation/styleguide/word_list.md +++ b/doc/development/documentation/styleguide/word_list.md @@ -17,6 +17,12 @@ For guidance not on this page, we defer to these style guides: +## @mention + +Try to avoid. Say "mention" instead, and consider linking to the +[mentions topic](../../../user/project/issues/issue_data_and_actions.md#mentions). +Don't use `code formatting`. + ## above Try to avoid extra words when referring to an example or table in a documentation page, but if required, use **previously** instead. diff --git a/doc/user/packages/dependency_proxy/index.md b/doc/user/packages/dependency_proxy/index.md index b1064ac9844..c76b0a6810f 100644 --- a/doc/user/packages/dependency_proxy/index.md +++ b/doc/user/packages/dependency_proxy/index.md @@ -250,6 +250,10 @@ hub_docker_quota_check: TOKEN=$(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq --raw-output .token) && curl --head --header "Authorization: Bearer $TOKEN" "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" 2>&1 ``` +## Use the NPM Dependency Proxy for NPM packages + +For information on this, see [Dependency Proxy](../npm_registry/#dependency-proxy). + ## Troubleshooting ### Dependency Proxy Connection Failure diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 201aeaa993f..fe7e6a0ea46 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -601,3 +601,8 @@ The GitLab npm repository supports the following commands for the npm CLI (`npm` - `npm view`: Show package metadata. - `yarn add`: Install an npm package. - `yarn update`: Update your dependencies. + +## Dependency Proxy + +The NPM Dependency Proxy for NPM packages isn't available. For more information, see +[this epic](https://gitlab.com/groups/gitlab-org/-/epics/3608). diff --git a/doc/user/profile/notifications.md b/doc/user/profile/notifications.md index 0a0411dc925..17b4251662e 100644 --- a/doc/user/profile/notifications.md +++ b/doc/user/profile/notifications.md @@ -143,7 +143,7 @@ For each project and group you can select one of the following levels: |:------------|:------------| | Global | Your global settings apply. | | Watch | Receive notifications for any activity. | -| On mention | Receive notifications when `@mentioned` in comments. | +| On mention | Receive notifications when [mentioned](../project/issues/issue_data_and_actions.md#mentions) in a comment. | | Participate | Receive notifications for threads you have participated in. | | Disabled | Turns off notifications. | | Custom | Receive notifications for custom selected events. | @@ -278,7 +278,7 @@ The participants are: - Authors of the design (can be multiple people if different authors have uploaded different versions of the design). - Authors of comments on the design. -- Anyone that is `@mentioned` in a comment on the design. +- Anyone that is [mentioned](../project/issues/issue_data_and_actions.md#mentions) in a comment on the design. ## Opt out of all GitLab emails diff --git a/doc/user/project/issues/issue_data_and_actions.md b/doc/user/project/issues/issue_data_and_actions.md index 2ef12cd1240..78dc6805f2b 100644 --- a/doc/user/project/issues/issue_data_and_actions.md +++ b/doc/user/project/issues/issue_data_and_actions.md @@ -199,7 +199,7 @@ You can mention a user or a group present in your GitLab instance with `@usernam unless they have disabled all [notifications](#notifications) in their user settings. This is controlled in the [notification settings](../../profile/notifications.md). -Mentions for yourself (the current logged in user) are highlighted +Mentions for yourself (the user currently signed in) are highlighted in a different color, which allows you to quickly see which comments involve you. Avoid mentioning `@all` in issues and merge requests, as it sends an email notification diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 59773c625c5..160c2314ded 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -234,16 +234,29 @@ different branch. > - Markdown preview [introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/18059) in [GitLab Free](https://about.gitlab.com/pricing/) 10.7. > - Support for pasting images [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22822) in [GitLab Free](https://about.gitlab.com/pricing/) 13.1. +> - Side-by-side Markdown preview [introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68274) in [GitLab Free](https://about.gitlab.com/pricing/) 14.3 -When you edit Markdown files in the Web IDE, you can preview your changes by -clicking the **Preview Markdown** tab above the file editor. The Markdown preview -supports [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown). +To edit Markdown files in the Web IDE: -You can also upload any local images by pasting them directly in the Markdown file. +1. Go to your repository, and navigate to the Markdown page you want to edit. +1. Select **Edit in Web IDE**, and GitLab loads the page in a tab in the editor. +1. Make your changes to the file. GitLab supports [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown). +1. When your changes are complete, select **Commit** in the left sidebar. +1. Add a commit message, select the branch you want to commit to, and select **Commit**. + +When editing, you can upload local images by pasting them directly in the Markdown file. The image is uploaded to the same directory and is named `image.png` by default. If another file already exists with the same name, a numeric suffix is automatically added to the filename. +There are two ways to preview Markdown content in the Web IDE: + +1. At the top of the file's tab, select **Preview Markdown** to preview the formatting + in your file. You can't edit the file in this view. + 1. To add more changes to the file, select **Edit**. +1. Right-click or use the keyboard shortcut `Command/Control + Shift + P` and + select **Preview Markdown** to toggle a live Markdown preview panel. + ## Live Preview > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/19764) in [GitLab Free](https://about.gitlab.com/pricing/) 11.2. diff --git a/doc/user/todos.md b/doc/user/todos.md index c367898cf59..f0601db0300 100644 --- a/doc/user/todos.md +++ b/doc/user/todos.md @@ -31,9 +31,9 @@ Many to-do items are created automatically. A to-do item is added to your To-Do List when: - An issue or merge request is assigned to you. -- You're `@mentioned` in the description or comment of an issue, merge request, - or epic. -- You are `@mentioned` in a comment on a commit or design. +- You're [mentioned](project/issues/issue_data_and_actions.md#mentions) in the description or + comment of an issue, merge request, or epic. +- You are mentioned in a comment on a commit or design. - The CI/CD pipeline for your merge request fails. - An open merge request cannot be merged due to conflict, and one of the following is true: diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 044dc4b8a75..08f108eb8e4 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -108,6 +108,11 @@ module Gitlab # Configures proxying of requests. def self.configure_proxy(proxy = ConnectionProxy.new(hosts)) ActiveRecord::Base.load_balancing_proxy = proxy + + # Populate service discovery immediately if it is configured + if service_discovery_enabled? + ServiceDiscovery.new(service_discovery_configuration).perform_service_discovery + end end def self.active_record_models diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 770d7a076ad..251961c8246 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -63,18 +63,14 @@ module Gitlab end def start - # We run service discovery once in the current thread so that the application's main thread - # does not race this thread to use the results of initial service discovery. - next_sleep_duration = perform_service_discovery - Thread.new do loop do + next_sleep_duration = perform_service_discovery + # We slightly randomize the sleep() interval. This should reduce # the likelihood of _all_ processes refreshing at the same time, # possibly putting unnecessary pressure on the DNS server. sleep(next_sleep_duration + rand(MAX_SLEEP_ADJUSTMENT)) - - next_sleep_duration = perform_service_discovery end end end diff --git a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb index 0434cec3288..a27341a3324 100644 --- a/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/service_discovery_spec.rb @@ -57,16 +57,13 @@ RSpec.describe Gitlab::Database::LoadBalancing::ServiceDiscovery do .and_yield end - it 'runs service discovery once before starting the worker thread' do - expect(service).to receive(:perform_service_discovery).ordered.and_return(5) - + it 'starts service discovery in a new thread' do expect(Thread).to receive(:new).ordered.and_call_original # Thread starts + expect(service).to receive(:perform_service_discovery).ordered.and_return(5) expect(service).to receive(:rand).ordered.and_return(2) expect(service).to receive(:sleep).ordered.with(7) # Sleep runs after thread starts - expect(service).to receive(:perform_service_discovery).ordered.and_return(1) - service.start.join end end diff --git a/spec/lib/gitlab/database/load_balancing_spec.rb b/spec/lib/gitlab/database/load_balancing_spec.rb index b220e541c74..6ec8e0516f6 100644 --- a/spec/lib/gitlab/database/load_balancing_spec.rb +++ b/spec/lib/gitlab/database/load_balancing_spec.rb @@ -212,6 +212,21 @@ RSpec.describe Gitlab::Database::LoadBalancing do expect(ActiveRecord::Base).to have_received(:load_balancing_proxy=) .with(Gitlab::Database::LoadBalancing::ConnectionProxy) end + + context 'when service discovery is enabled' do + let(:service_discovery) { double(Gitlab::Database::LoadBalancing::ServiceDiscovery) } + + it 'runs initial service discovery when configuring the connection proxy' do + allow(described_class) + .to receive(:configuration) + .and_return('discover' => { 'record' => 'foo' }) + + expect(Gitlab::Database::LoadBalancing::ServiceDiscovery).to receive(:new).and_return(service_discovery) + expect(service_discovery).to receive(:perform_service_discovery) + + described_class.configure_proxy + end + end end describe '.active_record_models' do diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index c08d56a7af4..26abc98656e 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -28,6 +28,7 @@ RSpec.describe Ci::Build do it { is_expected.to have_one(:deployment) } it { is_expected.to have_one(:runner_session) } + it { is_expected.to have_one(:trace_metadata) } it { is_expected.to validate_presence_of(:ref) } diff --git a/spec/models/ci/build_trace_metadata_spec.rb b/spec/models/ci/build_trace_metadata_spec.rb new file mode 100644 index 00000000000..42b9d5d34b6 --- /dev/null +++ b/spec/models/ci/build_trace_metadata_spec.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::BuildTraceMetadata do + it { is_expected.to belong_to(:build) } + it { is_expected.to belong_to(:trace_artifact) } + + it { is_expected.to validate_presence_of(:build) } +end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index 42d6e66b38b..ce2e6ba5e15 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -60,7 +60,7 @@ RSpec.describe Ci::RetryBuildService do artifacts_file artifacts_metadata artifacts_size commands resource resource_group_id processed security_scans author pipeline_id report_results pending_state pages_deployments - queuing_entry runtime_metadata].freeze + queuing_entry runtime_metadata trace_metadata].freeze shared_examples 'build duplication' do let_it_be(:another_pipeline) { create(:ci_empty_pipeline, project: project) } diff --git a/workhorse/.gitlab-ci.yml b/workhorse/.gitlab-ci.yml deleted file mode 100644 index 60d51b868b7..00000000000 --- a/workhorse/.gitlab-ci.yml +++ /dev/null @@ -1,83 +0,0 @@ -workflow: - rules: &workflow_rules - # For merge requests, create a pipeline. - - if: '$CI_MERGE_REQUEST_IID' - # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.). - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - # For tags, create a pipeline. - - if: '$CI_COMMIT_TAG' - # For stable branches, create a pipeline. - - if: '$CI_COMMIT_BRANCH =~ /^[\d-]+-stable$/' - -default: - image: golang:1.16 - tags: - - gitlab-org - -# Disable DIND for SAST because we need to execute a before_script in the gosec-sast job -variables: - SAST_DISABLE_DIND: "true" - -verify: - script: - - make verify - -changelog: - script: - - _support/check_changelog.sh - rules: - - if: '$CI_MERGE_REQUEST_IID' - -.test: - services: - - name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest - # Disable the hooks so we don't have to stub the GitLab API - command: ["/usr/bin/env", "GITALY_TESTING_NO_GIT_HOOKS=1", "/scripts/process-wrapper"] - alias: gitaly - variables: - GITALY_ADDRESS: "tcp://gitaly:8075" - script: - - go version - - apt-get update && apt-get -y install libimage-exiftool-perl - - make test - -test using go 1.15: - extends: .test - image: golang:1.15 - -test using go 1.16: - extends: .test - image: golang:1.16 - -test:release: - rules: - - if: '$CI_COMMIT_TAG' - script: - - git describe --exact-match - -include: - - template: Security/SAST.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml - -gosec-sast: - before_script: - - apk add make - - make install - rules: *workflow_rules - -gemnasium-dependency_scanning: - rules: *workflow_rules - -secret_detection: - rules: *workflow_rules - -code_navigation: - image: golang:latest - allow_failure: true - script: - - go get github.com/sourcegraph/lsif-go/cmd/lsif-go - - lsif-go - artifacts: - reports: - lsif: dump.lsif diff --git a/workhorse/.gitlab/CODEOWNERS b/workhorse/.gitlab/CODEOWNERS deleted file mode 100644 index 93ee023ee4a..00000000000 --- a/workhorse/.gitlab/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @jacobvosmaer-gitlab @nick.thomas @nolith @patrickbajao diff --git a/workhorse/CHANGELOG b/workhorse/CHANGELOG deleted file mode 100644 index b742affae07..00000000000 --- a/workhorse/CHANGELOG +++ /dev/null @@ -1,1137 +0,0 @@ -# Changelog for gitlab-workhorse - -## v8.65.0 - -### Fixed -- Fix long polling to default to 50 s instead of 50 ns - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/687 - -### Security -- Use URL.EscapePath() in upstream router - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -## v8.64.0 - -### Other -- Revert "Migrate to labkit error tracking" - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/685 - -## v8.63.0 - -### Added -- Accept more paths as Git HTTP - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/684 - -### Other -- Migrate error tracking from raven to labkit - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/671 - -## v8.62.0 - -### Added -- Add RubyGems registry upload route - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/680 - -### Fixed -- Cleanup Connection headers - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/678 - -## v8.61.0 - -### Fixed -- Revert "Support Git HTTP on toplevel repositories" - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/681 - -## v8.60.0 - -### Added -- Support Git HTTP on toplevel repositories - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/670 -- Update GoCloud to v0.21.1+ - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/675 - -### Changed -- Allow blank S3 regions to be used - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/677 - -## v8.59.0 - -### Fixed -- Image scaling: strip out iCCP chunks in PNG files - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/673 - -### Other -- Extract logging concerns into a separate module - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -### Security -- Reject unknown http methods - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -## v8.58.0 - -### Added -- Support alternate document root directory - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/626 - -### Fixed -- Fix uploader not returning 413 when artifact too large - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/663 -- Auto-register Prometheus metrics - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/660 - -### Other -- Do not resize when image is less than 8 bytes - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/666 - -## v8.57.0 - -### Added -- Add direct upload acceleration for requirements import - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -## v8.56.0 - -### Fixed -- Return 413 HTTP status for S3 uploads if max upload limit is reached - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/655 -- Fix EXIF cleaning for S3 compatible Object Storage - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -### Other -- Improve logging for image scaler - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/652 -- Update LabKit to v1.0.0 - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/659 - -## v8.55.0 - -### Added -- Add direct upload acceleration for metric images - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/653 - -### Fixed -- Image scaler: add success-client-cache status label - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/656 - -## v8.54.0 - -### Changed -- Don't reject image scaling requests based on file extension/format mismatch - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/651 - -### Other -- Rework image scaler test suite - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ -- Adjust image scaling latency buckets - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/643 -- Update raven-go and gocertifi packages - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/644 -- jaeger: limit operation cardinality by using route regex - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/648 - -### Performance -- Add support for conditional GETs for rescaled images - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/647 - -## v8.53.0 - -### Added -- Add route for Debian package uploads - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/610 - Contributed by Mathieu Parent - -### Fixed -- Don't log image scaler fail-overs as successes - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/636 - -### Other -- Exclude dot-files from "make fmt" target - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/641 -- Add max_processes Prometheus metric for image scaling - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/640 -- Simplify config handling in main() - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/634 -- Default MaxScalerProcs to num_cores / 2 - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/635 -- Add a total requests metric for image scaling - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/638 - -## v8.52.0 - -### Fixed -- Only generate CI artifact metadata for ZIP files - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/627 -- Fix typo in redis URL scheme - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/631 -- Restructure error handling in main() - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/633 - -### Other -- Include route regex identifier in structured logs - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/624 - -## v8.51.0 - -### Changed -- Allow configure image resizing params - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/615 - -### Fixed -- Fix processing lsif dump with repeating lines with inVs - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/621 - Contributed by Pavel Kuznetsov - -### Other -- Add CODEOWNERS with listed maintainers - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/623 - -## v8.50.0 - -### Added -- Update Gitaly module dependency - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/619 - -## v8.49.0 - -### Fixed -- Fix gitlab-resize-image bin installation - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/618 - -### Other -- Add image scaler duration histogram - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/614 -- Pass CORRELATION_ID env variable to resize image subprocesses - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/612 -- Simplify s3 session management code - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/616 -- Bump labkit dependency to get mutex profiling - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -## v8.48.0 - -### Changed -- Switch image scaler to a Go-only solution - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/603 - -### Other -- Push uploader control flow into objectstore package - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/608 - -## v8.47.0 - -### Added -- Add logging for local LSIF ZIP caching - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/579 -- Add project level route for Generic Packages uploads - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/596 - -### Changed -- Further simplify remote/local upload code - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/602 -- Experimental: Use strict content checks when resizing images - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/564 - -### Fixed -- Increase LSIF scanner buffer - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/609 -- Fix correlation IDs not being propagated in preauth check - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/607 - -### Other -- Reflect the actual duration of bootstrapping GitLab - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/605 - Contributed by Takuya Noguchi - -## v8.46.0 - -### Added -- Support Azure custom storage domains - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/593 - -## v8.45.0 - -### Added -- Reject upload when filesize exceeds MaximumSize returned by authorize endpoint - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -### Other -- Eliminate unnecessary code in GoCloud test stubs - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/590 -- Drop tests that check for log messages - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/591 - -## v8.44.0 - -### Fixed -- Fix objectstore.uploader.uploadError race - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/583 -- Silence errors when Azure objects have aleady been deleted - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/585 -- Fix race condition in httprs test - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/584 - -### Performance -- Remove an in-memory buffer for LSIF transformation - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/586 - -## v8.43.0 - -### Changed -- Remove ProcessReferences flag - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/543 - -### Fixed -- Fix nil pointer exception when no object storage config is defined - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/565 - -## v8.42.0 - -### Added -- Resize images on-demand with `gm convert` - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/546 - -## v8.41.0 - -### Added -- Add Azure blob store support - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/555 - -## v8.40.0 - -### Added -- Add project level route for conan package uploads - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/558 - -### Other -- Refactor uploaders to use different upload strategies - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/553 - -## v8.39.0 - -### Fixed -- Fix HTTP Range Requests not working on some S3 providers - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/549 - -### Other -- Vendor httprs module - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/550 - -### Performance -- Cache references in file - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/544 - -## v8.38.0 - -### Added -- Added configuration option PropagateCorrelationID - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/529 - Contributed by Mahmoud Rahbar Azad -- Add support for AWS S3 Server Side Encryption (SSE-KMS) - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/537 - -### Changed -- Drop Go v1.12 support - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/532 - -## v8.37.0 - -- No changes. -## v8.36.0 - -- No changes. -## v8.35.0 - -### Fixed -- Fix Content-Length set prior to SendUrl injection - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/496 - Contributed by Georges-Etienne Legendre - -## v8.34.0 - -### Added -- Support Workhorse directly uploading files to S3 - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/466 - -### Fixed -- Disable compression for open archive - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/508 - Contributed by Georges-Etienne Legendre - -### Other -- Add configuration to support an S3 client inside Workhorse - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/516 -- Refactor Preparer and SaveFileOpts handling - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/515 - -## v8.33.0 - -### Added -- Add routes for Group import via the UI - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/493 - -### Fixed -- Gather gitlab-zip-cat/metadata stderr and log output in the current context - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/498 - Contributed by Georges-Etienne Legendre - -## v8.32.1 - -### Security -- Limit memory footprint of a command that generates ZIP artifacts metadata - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -## v8.32.0 - -### Added -- Process LSIF document before sending it to GitLab - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/492 -- Delay PostUploadPack response until request is fully read - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/494 - -## v8.31.1 - -### Security -- Limit memory footprint of a command that generates ZIP artifacts metadata - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -## v8.31.0 - -### Added -- Add a signed field on upload requests containing all the workhorse parameters - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/490 - -### Other -- Add automatic changelog generation - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/484 - -## v8.30.2 - -### Security -- Limit memory footprint of a command that generates ZIP artifacts metadata - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -v 8.30.1 - -- Sign artifact multipart fields in Workhorse - -v 8.30.0 - -- Proxy ActionCable websocket connection !454 - -v 8.29.0 - -- Bump Labkit version to support Profiler sample versioning !479 - -v 8.28.0 - -- Reject parameters that override upload fields -- PyPi - Object storage upload route for package files !474 - -v 8.27.0 - -- Remove Set-Cookie header from archive and raw blob responses !475 - -v 8.26.0 - -- Add route for project imports direct upload via UI !470 - -## v8.25.3 - -### Security -- Limit memory footprint of a command that generates ZIP artifacts metadata - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/ - -### Other -- Add automatic changelog generation - https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/484 - -v 8.25.2 - -- Sign artifact multipart fields in Workhorse - -v 8.25.1 - -- Reject parameters that override upload fields - -v 8.25.0 - -- Add route for project imports direct upload !459 - -v 8.24.0 - -- Support Stackdriver Profiler through Labkit !461 - -v 8.23.0 - -- Don't set Cache-Control header for git archives !462 - -v 8.22.0 - -- Bump the version of golang.org/x/sys !456 -- Add friendly development error page for 502 !453 - -v 8.21.2 - -- Sign artifact multipart fields in Workhorse - -v 8.21.1 - -- Reject parameters that override upload fields - -v 8.21.0 - -- Add route for group imports direct upload !455 - -v 8.20.2 - -- Sign artifact multipart fields in Workhorse - -v 8.20.1 - -- Reject parameters that override upload fields - -v 8.20.0 - -- Sign file upload requests modified by workhorse - -v 8.19.0 - -- Use multipart uploads for nuget packages !451 - -v 8.18.0 - -- Allow inline content disposition for pdf files !446 -- Update environment websocket route !449 - -v 8.17.0 - -- Add labkit monitoring for build metrics !440 -- Log duration_ms when Workhorse hits a bad gateway !445 - -v 8.16.0 - -- Ignore CompleteMultipartUpload ETag !438 -- Add NuGet route for package uploads !441 -- Upgrade Gitaly client to v1.74.0 !443 -- Set a time limit on git upload-pack requests - -v 8.15.0 - -- Object store case insensitive ETag comparison !434 -- Upgrade gitaly to 1.68.0 !435 - -v 8.14.1 - -- Set a time limit on git upload-pack requests - -v 8.14.0 - -- Keep HTTP 1.0 cache headers from sendurl proxies !431 - -v 8.13.0 - -- Preserve original HTTP cache headers when proxying with sendurl !428 - -v8.12.0 - -- Fix health checks routes incorrectly intercepting errors !424 -- Simplify badgateway RoundTripper !425 - -v8.11.0 - -- Accelerate GraphQL uploads !403 -- Add route for handling Conan package uploads !412 -- Accelerate wiki attachments !422 - -v8.10.1 - -- Set a time limit on git upload-pack requests - -v8.10.0 - -- Use accelerated uploads for users/personal snippets -- Fix typo in keywatcher prometheus metrics !420 - -v8.9.0 - -- Update Gitaly library code to v1.57.0 !407 -- Replace govendor with go mod !411 -- Support gzip compression for Git info/refs !404 -- Add prometheus counter for Gitaly connection stubs !414 -- Support passing on Gitaly feature flags !410 - -v8.8.1 - -- Use accelerated uploads for users/personal snippets - -v8.8.0 - -- Filter title, description, text, and body from logs !402 -- Remove redirections from Terminal to Channel !397 -- Add option to set Sentry environment !396 - -v8.7.1 - -- Use accelerated uploads for users/personal snippets - -v8.7.0 - -- Don't log http.ErrAbortHandler panics in sentry !392 - -v8.6.0 - -- Add new endpoint to add support to proxy websocket requests to build's services !370 - -v8.5.2 - -- Don't log http.ErrAbortHandler panics in sentry !392 - -v8.5.1 - -- Remove duplicate X-Request-Id response header !384 - -v8.5.0 - -- Replace terminal terminology to channel !382 - -v8.4.0 - -- Adds X-Request-Id response header for Workhorse !363 -- Change content type detection header size to 4k !366 -- Allow unknown fields in jsonpb gitaly-proto messages !367 -- Filter `sharedSecret` param from Jira !369 -- Get git-archive with GetArchiveRequest !375 - -v8.3.3 - -- Preserve orientation when removing EXIF - -v8.3.2 - -- Remove EXIF from JPEG/TIFF images - -v 8.3.1 - -- Update gitaly-proto to 1.10.0 !363 - -v 8.3.0 - -- Count ^/-/ requests separately in prometheus !355 -- Statically link jaeger into Workhorse by default !359 -- Support encoded Content-Disposition fields !360 - -v 8.2.0 - -- Sign LFS upload requests that have been handled by workhorse -- Fixed svg recognition to get the proper content type !353 - -v 8.1.1 - -- Sign LFS upload requests that have been handled by workhorse - -v 8.1.0 - -- Upgrade the gitaly client to v1.13.0 (includes TLS support) !351 -- Update gitaly-proto to 0.124.0 !331 -- Add distributed tracing with LabKit !325 - -v 8.0.4 - -- Preserve orientation when removing EXIF - -v 8.0.3 - -- Remove EXIF from JPEG/TIFF images - -v 8.0.2 - -- Fixed svg recognition to get the proper content type !353 - -v 8.0.1 - -- Sign LFS upload requests that have been handled by workhorse - -v 8.0.0 - -- Remove local git archive support !304 -- Remove local git diff handling !345 -- Remove local git format-patch handling !346 -- Remove RepoPath from the API response - -v 7.6.1 - -- Sign LFS upload requests that have been handled by workhorse - -v 7.6.0 - -- Rename correlation-id structured logging field to correlation_id !343 -- Remove local git receive-pack implementation !326 -- Remove curl from sendfile_test.go !344 -- Update README.md usage example !342 - -v 7.5.1 - -- Rename correlation-id structured logging field to correlation_id !343 - -v 7.5.0 - -- Add proxy layer to calculate content type and disposition headers !335 - -v 7.4.0 - -- Strip port and include remote IP in access logs !337 - -v 7.3.0 - -- Redact sensitive url params as in Rails - -v 7.2.1 - -- Extract correlation code out to the LabKit project !323 -- Log X-Forwarded-For IPs when UNIX domain sockets are in use !324 - -v 7.2.0 - -- Update CI matrix to go1.10 + go1.11 and fix ResponseWriter bugs !309 -- Add support for Redis URLs (redis:// and rediss://) in Workhorse !321 - -v 7.1.4 - -- Sign LFS upload requests that have been handled by workhorse - -v 7.1.3 - -- Redact sensitive url params as in Rails - -v 7.1.1 - -Bad release, use 7.2.0 instead. - -v 7.1.0 - -- Add structured logFormat for text based logging !275 -- Run make fmt on master !306 -- Allow to configure `BUILD_DIR` and `TARGET_DIR` !308 -- Resolve "Rework test suite to allow dead code to be removed" !307 -- Update Prometheus vendoring !305 -- General vendoring cleanup !310 -- Remove Go 1.8 support !314 -- Remove unused 'body' argument !315 -- Refactor badgateway to use standardlib interfaces !316 -- Pass Correlation-Ids down to backend systems !311 -- Don't fail if /home/git/repositories already exists in Gitaly container !317 - -v 7.0.1 - -- Redact sensitive url params as in Rails - -v 7.0.0 - -- Use the new Gitaly auth scheme (v2) !298 - -v 6.1.2 - -- Redact sensitive url params as in Rails - -v 6.1.1 - -- Allow custom error messages to pass through to Rails !300 - -v 6.1.0 - -- Support adding PUT headers for object storage from Rails !297 - -v 6.0.0 - -- Accelerate Maven artifact repository uploads !283 - -v 5.2.0 - -- Populate Git Protocol !276 -- Add support for GitConfigOptions required for git-receive-pack command !281 - - -v 5.1.0 - -- Log using correlation-id bound to the incoming request !258 -- Prevent uploading two files as artifacts in single request !273 -- Prometheus instrumentation !279 - -v 5.0.0 - -- Update httprs for broken range implementations !266 -- Direct Upload for User Uploads !265 - -v 4.3.1 - -- Objectstorage ETag checking !263 - -v 4.3.0 - -- Multipart upload support !257 -- Make external commands extend the environment !261 - -v 4.2.1 - -- Fix objectstore error shadowing !259 - -v 4.2.0 - -- Guess RemoteAddr from X-Forwarded-For !254 - -v 4.1.0 - -- Add websocket route for web terminal access to CI jobs !234 -- Remove RepoPath check on Git HTTP !244 -- Artifacts and Uploads must allow Objects Storage only requests !247 -- Bridge between Gitaly and GitLab for a new repository snapshot endpoint !248 -- Update gitaly proto !249 - -v 4.0.0 - -- Handle Object Store upload in upload.HandleFileUploads !238 -- More consistent API naming. ObjectStore -> RemoteObject !240 - -v3.8.0 - -- Add structured logging !236 - -v3.7.0 - -- Add option to send file uploads straight to object storage !227 -- Allow sending Git archives with file names other than 'archive' !232 -- Unify uploads handling under filestore package !230 - -v3.6.0 - -- Introduce a `send-url:` method that allows to serve remote HTTP/GET file, like S3-based file !228 - -v3.5.1 - -- Use grpc-go 1.9.1 (!225) -- Update gitaly stream and dial library functions (!224) - -v3.5.0 - -- Add option to disable Git archive caching !222 - -v3.4.0 - -- Track Gitaly Connections in Prometheus !211 -- Run test suite on Go 1.9 !213 -- Remove repo disk check !218 - -v3.3.1 - -- Fix "net/http: request canceled" errors in gitlab-zip-cat !208 - -v3.3.0 - -- Ban context.Background !201 -- Respect the ShowAllRefs flag in git upload-pack and info-refs !203 -- Upgrade grpc to v1.7.1, protobuf to latest !207 - -v3.2.0 - -- Implement Gitaly call for archive requests !199 -- Re-use client.Dial from gitaly !194 -- Respect GL_USERNAME !192 -- Update BurntSushi/toml !195 -- Add Redis error counters !197 -- Migrate Send{Diff,Patch} to Gitaly !200 - -v3.1.0 - -- Add histograms to routes !184 -- Gitaly deprecations and replacements !186, !187, !189 -- Enable CI long polling by default !188 -- Refactor Git archive creation !190 - -v3.0.0 - -- Use GetBlob RPC instead of TreeEntry RPC for serving blobs !182 - -v2.3.0 -- Improve gitaly info refs error message !172 -- Migrate GetBlob to Gitaly !174 -- Drop support for Go <1.8 !176 -- Add some tests for gzipped assets !177 -- Use reader/writer from gitaly streamio !178 -- Use http.Request contexts for Gitaly calls !179 -- Allow to access remote archive !180 - -v2.2.0 -- Add support for token authentication on Gitaly requests -- Update gitaly-proto library to 0.9.0 - -v2.1.1 -- Bug fix and counters for static error pages - -v2.1.0 -- Remove chatty ErrorPage log message -- Filter query-string secrets out of logged URLs -- Suggest better default for prometheus port -- Add internal upload to external storage -- Prometheus metrics for senddata and git archive cache - -v2.0.1 -- Support GL_REPOSITORY from API and pass it to Gitaly on ReceivePack - -v2.0.0 - -- Fix gRPC stream resource leak !158, !160 -- Don't append error messages to Git HTTP responses !157 -- Drop support for old Gitaly fields in Git API response !152 - -v1.4.3 - -- Support forwarding Git HTTP POST data to Gitaly !143 -- Pass more Gitaly 'Repository' fields on from gitlab-rails !147 -- Support insecure TCP connections to Gitaly !150 - -v1.4.2 - -- Return 500 from GET /info/refs if possible !145 - -v1.4.1 - -- Fix several Redis integration bugs !137, !140 -- Fix race conditions in Redis tests !136 -- Don't follow HTTP redirects on internal API !134 -- Support /api/v4 for CI !133 -- Don't spam logs with CI queueing messages (Marcin Biegała) !127 - -v1.4.0 - -- Integrate with Gitaly via gRPC !119 -- Buffer git receive-pack responses in tempfiles !123 -- Use stdlib to copy stdin/stdout of git subprocesses !118 -- Terminal session timeouts !107 -- Redis integration EXPERIMENTAL !112 -- CI notifications via Redis EXPERIMENTAL !128 -- More CI queue metrics !122 - -v1.3.0 - -- Fix stalled HTTP fetches with large payloads. !110 -- Correctly parse content types in HTTP requests and responses !114 -- Catch _all_ multipart NextPart() errors. !108 -- Replace 'gitlab_workhorse_artifacts_upload_*' with labeled version of - 'gitlab_workhorse_multipart_upload_*'. !106 -- Allow GET /info/refs to be proxied to Gitaly. !105 -- Set correct value of X-Forwarded-For header in PreAuthorize request. !104 -- Allow nested namespaces in git URLs. !80 - -v1.2.1 - -- More Prometheus metrics -- Hide 502 internal errors from text-mode clients -- Buffer internal API responses up to a 32kB hard limit - -v1.2.0 - -- Add terminal websocket proxy endpoint -- Rewrite all incoming multipart requests: write 'file' parts to tempfiles - -v1.1.1 - -- Restrict effect of API rate limiting to /ci/api/v1/builds/register.json - -v1.1.0 - -- Prometheus metrics listener via `-prometheusListenAddr` option -- Tell NGINX to not buffer Git HTTP responses etc. with X-Accel-Buffering -- Fix double content type bug on archive downloads - -v1.0.0 - -- Workhorse is now v1.0.0, according to Semantic Versioning. No breaking - changes were made. -- Add support for logging to file, and logfile rotation with SIGHUP. -- Improve error messages. - -v0.8.5 - -Simplify revspec for 'git format-patch'. - -v0.8.4 - -Fix Go 1.5 compatibility broken in 0.8.3. Update CI configuration so -that tests run on Go 1.5, 1.6 and 1.7 (was only 1.6 before). - -v0.8.3 - -Add rate-limiting feature for /api requests (disabled by default). -Suppress non-zero exit code error from git-upload-pack during shallow -Git clone (only affects logging and Sentry). Don't treat EEXIST as an -error during git archive finalization. - -v0.8.2 - -Recognize more archive formats in git.SendArchive. Make 502 errors -(failed proxy requests to Unicorn) easier to recognize in Sentry. - -v0.8.1 - -Add Sentry (raven-go) for remote error tracking. - -v0.8.0 - -Add JWT signed communication between gitlab-workhorse and gitlab-rails. - -v0.7.11 - -Fix 'nil dereference' crash on Go 1.7 when parsing authBackend -parameter. Fix 'hard-wire backend host' crashes. - -v0.7.10 - -Fix typo in metrics header name. - -v0.7.9 - -Hard-wire backend host when using TCP. - -v0.7.8 - -Send artifact zip file entries via the 'senddata' mechanism. - -v0.7.7 - -Add the protocol used (HTTP) to each gitCommand call in order to check -for restricted protocol access on GitLab's side. - -v0.7.6 - -Add the capability to inject `git format-patch` output. - -v0.7.5 - -Add the capability to inject `git diff` output as HTTP response bodies -(@zj). - -v0.7.4 - -Pass a timestamp when forwarding requests to Rails. Hopefully this -will give us insight into Unicorn queueing behavior. - -v0.7.3 - -Revert 'buffer Git HTTP responses'. Set default listen socket -permissions to world read/writeable. - -v0.7.2 DO NOT USE - -Integrate with GOPATH during development (remove relative imports -etc.). Buffer Git HTTP responses so that we may return an error if the -local command fails early. - -Update: the 'buffer Git HTTP responses' change in 0.7.2 is BAD, it -breaks shallow Git clone. Don't use 0.7.2! - -v0.7.1 - -Set Content-Length (retrieved from Git) on raw blob data responses. - -v0.7.0 - -Start using a 'v' prefix on the version string. - -0.6.5 - -Inject 'git archive' data the same way as Git blob data. - -0.6.4 - -Increase default ProxyHeadersTimeout to 5 minutes. Fix injecting raw -blobs for /api/v3 requetsts. - -0.6.3 - -Add support for sending Git raw git blobs via gitlab-workhorse. - -0.6.2 - -We now fill in missing directory entries in archize zip metadata -files; also some other minor changes. - -0.6.1 - -Add support for generating zip artifacts metadata and serving single -files from zip archives. - -Gitlab-workhorse now consists of multiple executables. We also fixed a -routing bug introduced by the 0.6.0 refactor that broke relative URL -support. - -0.6.0 - -Overhauled the source code organization; no user-facing changes -(intended). The application code is now split into Go 'packages' -(modules). As of 0.6.0 gitlab-workhorse requires Go 1.5 or newer. - -0.5.4 - -Fix /api/v3/projects routing bug introduced in 0.5.2-0.5.3. - -0.5.3 - -Fixes merge error in 0.5.2. - -0.5.2 (broken!) - -- Always check with upstream if files in /uploads/ may be served -- Fix project%2Fnamespace API project ID's -- Prevent archive zombies when using gzip or bzip2 -- Don't show pretty error pages in development mode - -0.5.1 - -Deprecate -relativeURLRoot option, use -authBackend instead. - -0.5.0 - -Send ALL GitLab requests through gitlab-workhorse. - -0.4.2 - -Return response to client when uploading Git LFS object. - -0.4.1 - -Add support for Build Artifacts and Git LFS. The GitLab-Workhorse -offloads file uploading and downloading by providing support for -rewriting multipart form data and X-Sendfile. - -Other changes: -- add header Gitlab-Workhorse to all requests to indicate from where - they originated - -0.4.0 - -Rename the project to gitlab-workhorse. The old name had become too -specific. - -Other changes: - -- pass LD_LIBRARY_PATH to Git commands -- accomodate broken HTTP clients by spelling 'Www-Authenticate' as - 'WWW-Authenticate' - -0.3.1 - -Add support for Unix domain socket connections to the authBackend. - -0.3.0 - -In 0.3.0 we also handle 'git archive' downloads for GitLab 8.1+. -This has lead to some breaking API changes, making 0.3.0 incompatible -with GitLab 8.0. We now expect the 'auth backend' (GitLab) to -provide us with much more information about each request, such as -the path on disk to the Git repository the client is requesting. -This makes the REPO_ROOT command line argument obsolete. - -0.2.14 - -This is the last version that works with GitLab 8.0. diff --git a/workhorse/CONTRIBUTING.md b/workhorse/CONTRIBUTING.md deleted file mode 100644 index ef9785b9b6c..00000000000 --- a/workhorse/CONTRIBUTING.md +++ /dev/null @@ -1,46 +0,0 @@ -## Contributing - -Thank you for your interest in contributing to this GitLab project! We welcome -all contributions. By participating in this project, you agree to abide by the -[code of conduct](#code-of-conduct). - -## Contributor license agreement - -By submitting code as an individual you agree to the [individual contributor -license agreement][individual-agreement]. - -By submitting code as an entity you agree to the [corporate contributor license -agreement][corporate-agreement]. - -## Code of conduct - -As contributors and maintainers of this project, we pledge to respect all people -who contribute through reporting issues, posting feature requests, updating -documentation, submitting pull requests or patches, and other activities. - -We are committed to making participation in this project a harassment-free -experience for everyone, regardless of level of experience, gender, gender -identity and expression, sexual orientation, disability, personal appearance, -body size, race, ethnicity, age, or religion. - -Examples of unacceptable behavior by participants include the use of sexual -language or imagery, derogatory comments or personal attacks, trolling, public -or private harassment, insults, or other unprofessional conduct. - -Project maintainers have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct. Project maintainers who do not follow the -Code of Conduct may be removed from the project team. - -This code of conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. - -Instances of abusive, harassing, or otherwise unacceptable behavior can be -reported by emailing contact@gitlab.com. - -This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant], version 1.1.0, -available at [http://contributor-covenant.org/version/1/1/0/](http://contributor-covenant.org/version/1/1/0/). - -[contributor-covenant]: http://contributor-covenant.org -[individual-agreement]: https://docs.gitlab.com/ee/legal/individual_contributor_license_agreement.html -[corporate-agreement]: https://docs.gitlab.com/ee/legal/corporate_contributor_license_agreement.html diff --git a/workhorse/LICENSE b/workhorse/LICENSE deleted file mode 100644 index 76a44f8bb70..00000000000 --- a/workhorse/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015-2017 GitLab B.V. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/workhorse/Makefile b/workhorse/Makefile index 0c44306554a..2bfaf5913c6 100644 --- a/workhorse/Makefile +++ b/workhorse/Makefile @@ -2,9 +2,6 @@ PREFIX=/usr/local PKG := gitlab.com/gitlab-org/gitlab/workhorse BUILD_DIR ?= $(CURDIR) TARGET_DIR ?= $(BUILD_DIR)/_build -TARGET_SETUP := $(TARGET_DIR)/.ok -BIN_BUILD_DIR := $(TARGET_DIR)/bin -COVERAGE_DIR := $(TARGET_DIR)/cover VERSION_STRING := $(shell git describe) ifeq ($(strip $(VERSION_STRING)),) VERSION_STRING := v$(shell cat VERSION) @@ -32,53 +29,40 @@ endef .PHONY: all all: clean-build $(EXE_ALL) -$(TARGET_SETUP): - $(call message,"Setting up target directory") - rm -rf "$(TARGET_DIR)" - mkdir -p "$(TARGET_DIR)" - touch "$(TARGET_SETUP)" - .PHONY: gitlab-resize-image -gitlab-resize-image: $(TARGET_SETUP) +gitlab-resize-image: $(call message,Building $@) $(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@ .PHONY: gitlab-zip-cat -gitlab-zip-cat: $(TARGET_SETUP) +gitlab-zip-cat: $(call message,Building $@) $(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@ .PHONY: gitlab-zip-metadata -gitlab-zip-metadata: $(TARGET_SETUP) +gitlab-zip-metadata: $(call message,Building $@) $(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@ .PHONY: gitlab-workhorse -gitlab-workhorse: $(TARGET_SETUP) +gitlab-workhorse: $(call message,Building $@) $(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG) .PHONY: install -install: $(EXE_ALL) +install: $(EXE_ALL) $(call message,$@) mkdir -p $(DESTDIR)$(PREFIX)/bin/ cd $(BUILD_DIR) && $(INSTALL) $(EXE_ALL) $(DESTDIR)$(PREFIX)/bin/ .PHONY: test -test: $(TARGET_SETUP) prepare-tests +test: prepare-tests $(call message,$@) @go test -tags "$(BUILD_TAGS)" ./... @echo SUCCESS -.PHONY: coverage -coverage: $(TARGET_SETUP) prepare-tests - $(call message,$@) - @go test -tags "$(BUILD_TAGS)" -cover -coverprofile=test.coverage ./... - go tool cover -html=test.coverage -o coverage.html - rm -f test.coverage - .PHONY: clean -clean: clean-workhorse clean-build +clean: clean-workhorse clean-build $(call message,$@) rm -rf testdata/data testdata/scratch @@ -87,20 +71,6 @@ clean-workhorse: $(call message,$@) rm -f $(EXE_ALL) -.PHONY: check-version -check-version: - @test -n "$(VERSION)" || (echo "VERSION not set." ; exit 1) - -.PHONY: tag -tag: check-version - $(call message,$@) - sh _support/tag.sh "$(VERSION)" - -.PHONY: signed_tag -signed_tag: check-version - $(call message,$@) - TAG_OPTS=-s sh _support/tag.sh "$(VERSION)" - .PHONY: clean-build clean-build: $(call message,$@) @@ -121,18 +91,18 @@ testdata/scratch: verify: lint vet detect-context detect-assert check-formatting staticcheck deps-check .PHONY: lint -lint: $(TARGET_SETUP) +lint: $(call message,Verify: $@) go install golang.org/x/lint/golint @_support/lint.sh ./... .PHONY: vet -vet: $(TARGET_SETUP) +vet: $(call message,Verify: $@) @go vet ./... .PHONY: detect-context -detect-context: $(TARGET_SETUP) +detect-context: $(call message,Verify: $@) _support/detect-context.sh @@ -142,7 +112,7 @@ detect-assert: _support/detect-assert.sh .PHONY: check-formatting -check-formatting: $(TARGET_SETUP) install-goimports +check-formatting: install-goimports $(call message,Verify: $@) @_support/fmt.sh check @@ -150,7 +120,7 @@ check-formatting: $(TARGET_SETUP) install-goimports # Additionally, megacheck will not return failure exit codes unless explicitly told to via the # `-simple.exit-non-zero` `-unused.exit-non-zero` and `-staticcheck.exit-non-zero` flags .PHONY: staticcheck -staticcheck: $(TARGET_SETUP) +staticcheck: $(call message,Verify: $@) go install honnef.co/go/tools/cmd/staticcheck @ $(GOBIN)/staticcheck -go $(MINIMUM_SUPPORTED_GO_VERSION) ./... @@ -158,12 +128,12 @@ staticcheck: $(TARGET_SETUP) # In addition to fixing imports, goimports also formats your code in the same style as gofmt # so it can be used as a replacement. .PHONY: fmt -fmt: $(TARGET_SETUP) install-goimports +fmt: install-goimports $(call message,$@) @_support/fmt.sh .PHONY: goimports -install-goimports: $(TARGET_SETUP) +install-goimports: $(call message,$@) go install golang.org/x/tools/cmd/goimports diff --git a/workhorse/PROCESS.md b/workhorse/PROCESS.md index cf29b23b2c0..1231ab3968c 100644 --- a/workhorse/PROCESS.md +++ b/workhorse/PROCESS.md @@ -4,154 +4,14 @@ GitLab-Workhorse has the following maintainers: +- Patrick Bajao `@patrickbajao` +- Alessio Caiazza `@nolith` - Nick Thomas `@nick.thomas` - Jacob Vosmaer `@jacobvosmaer-gitlab` -- Alessio Caiazza `@nolith` -This list is defined at https://about.gitlab.com/team/. - -## Changelog - -GitLab-Workhorse keeps a changelog which is generated when a new release -is created. The changelog is generated from entries that are included on each -merge request. To generate an entry on your branch run: -`_support/changelog "Change descriptions"`. - -After the merge request is created, the ID of the merge request needs to be set -in the generated file. If you already know the merge request ID, run: -`_support/changelog -m "Change descriptions"`. - -Any new merge request must contain either a new entry or a justification in the -merge request description why no changelog entry is needed. +This authoritative source for this list is https://about.gitlab.com/team/. ## Merging and reviewing contributions Contributions must be reviewed by at least one Workhorse maintainer. The final merge must be performed by a maintainer. - -## Releases - -> Below we describe the legacy release process, from when Workhorse -> had its own repository. These instructions are still useful for -> security backports. - -New versions of Workhorse can be released by one of the Workhorse -maintainers. The release process is: - -- pick a release branch. For x.y.0, use `master`. For all other - versions (x.y.1, x.y.2 etc.) , use `x-y-stable`. Also see [below](#versioning) -- run `make tag VERSION=x.y.z"` or `make signed_tag VERSION=x.y.z` on the release branch. This will - compile the changelog, bump the VERSION file, and make a tag matching it. -- push the branch and the tag to gitlab.com -- the new version will only be deployed to `gitlab.com` if [`GITLAB_WORKHORSE_VERSION`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/GITLAB_WORKHORSE_VERSION) is updated accordingly; - if applicable, please remind the person who originally asked for a new release to make this change - (the MR should include a link back to the [version tag](https://gitlab.com/gitlab-org/gitlab-workhorse/-/tags) and a copy of the changelog) -- the person who updates GITLAB_WORKHORSE_VERSION should also run `scripts/update-workhorse` after commiting the new GITLAB_WORKHORSE_VERSION. If they forget they will be reminded by CI. - -## Security releases - -Workhorse is included in the packages we create for GitLab, and each version of -GitLab specifies the version of Workhorse it uses in the `GITLAB_WORKHORSE_VERSION` -file, so security fixes in Workhorse are tightly coupled to the [general security release](https://about.gitlab.com/handbook/engineering/workflow/#security-issues) -workflow, with some elaborations to account for the changes happening across two -repositories. In particular, the Workhorse maintainer takes responsibility for -creating new patch versions of Workhorse that can be used in the security -release. - -As security fixes are backported three releases in addition to master, and -changes need to happen across two repositories, up to eight merge requests, and -four Workhorse releases, can be required to fix a security issue in Workhorse. -This is a lot of overhead, so in general, it is better to fix security issues -without changing Workhorse. Where changes **are** necessary, this section -documents the necessary steps. - -If you're working on a security fix in Workhorse, you need two sets of merge -requests: - -* The fix itself, in the `gitlab-org/security/gitlab-workhorse` repository -* A merge request to change the version of workhorse included in the GitLab - security release, in the `gitlab-org/security/gitlab` repository. - -If the Workhorse maintainer isn't also a GitLab maintainer, reviews will need to -be split across several people. If changes to GitLab **code** are required in -addition to the change of Workhorse version, they both happen in the same merge -request. - -Start by creating a single merge request targeting `master` in Workhorse. Ensure -you include a changelog! If code changes are needed in GitLab as well, create a -GitLab merge request targeting `master` at this point, but don't worry about the -`GITLAB_WORKHORSE_VERSION` file yet. - -Once the changes have passed review, the Workhorse maintainer will determine the -new versions of Workhorse that will be needed, and communicate that to the -author. To do this, examine the `GITLAB_WORKHORSE_VERSION` file on each GitLab -stable branch; for instance, if the security release consisted of GitLab -versions `12.10.1`, `12.9.2`, `12.8.3`, and `12.7.4`, we would see the following: - -``` -gitlab$ git fetch security master 12-10-stable-ee 12-9-stable-ee 12-8-stable-ee 12-7-stable-ee` -gitlab$ git show refs/remotes/security/master:GITLAB_WORKHORSE_VERSION -8.30.1 -gitlab$ git show refs/remotes/security/12-10-stable-ee:GITLAB_WORKHORSE_VERSION -8.30.1 -gitlab$ git show refs/remotes/security/12-9-stable-ee:GITLAB_WORKHORSE_VERSION -8.25.2 -gitlab$ git show refs/remotes/security/12-8-stable-ee:GITLAB_WORKHORSE_VERSION -8.21.2 -gitlab$ git show refs/remotes/security/12-7-stable-ee:GITLAB_WORKHORSE_VERSION -8.21.2 -``` - -In this example, there are three distinct Workhorse stable branches to be -concerned with, plus Workhorse master: `8-30-stable`, `8-25-stable`, and -`8-21-stable`, and we can predict that we are going to need to create Workhorse -releases `8.30.2`, `8.25.3`, and `8.21.3`. - -The author needs to create a merge request targeting each Workhorse stable -branch, and verify that the fix works once backported. They also need to create -(or update, if they already exist) GitLab merge requests, setting the -`GITLAB_WORKHORSE_VERSION` file to the predicted workhorse version, and assign -all the MRs back to the appropriate maintainer(s). The pipeline for the GitLab -MRs will fail until the Workhorse releases have been tagged; you can use the -`=workhorse_branch_name` syntax in the `GITLAB_WORKHORSE_VERSION` file to verify -that the MRs interact as expected, if necessary. - -Once all involved maintainers are happy with the overall change, the Workhorse -maintainer will merge each of the Workhorse MRs and generate new Workhorse -releases from the stable branches. The tags will be present on the `security` -mirror and `dev.gitlab.org` **only** at this point. - -Once the Workhorse tags exist, the GitLab maintainer ensures that all the GitLab -MRs are green and assigns those MRs on to the release bot. - -The release managers merge the GitLab MRs, tag GitLab releases that reference -the new Workhorse tags, and release them in the usual way. - -Once the security release is done, the Workhorse maintainer is responsible for -syncing the changes to the `gitlab-org/gitlab-workhorse` repository. Push the -changes to `master`, the new tags, and all the changes to the stable branches. - -This process is quite involved, very manual, and extremely error-prone; work is -ongoing on automating it. - -## Versioning - -Workhorse uses a variation of SemVer. We don't use "normal" SemVer -because we have to be able to integrate into GitLab stable branches. - -A version has the format MAJOR.MINOR.PATCH. - -- Major and minor releases are tagged on the `master` branch -- If the change is backwards compatible, increment the MINOR counter -- If the change breaks compatibility, increment MAJOR and set MINOR to `0` -- Patch release tags must be made on stable branches -- Only make a patch release when targeting a GitLab stable branch - -This means that tags that end in `.0` (e.g. `8.5.0`) must always be on -the master branch, and tags that end in anthing other than `.0` (e.g. -`8.5.2`) must always be on a stable branch. - -> The reason we do this is that SemVer suggests something like a -> refactoring constitutes a "patch release", while the GitLab stable -> branch quality standards do not allow for back-porting refactorings -> into a stable branch. diff --git a/workhorse/_support/changelog b/workhorse/_support/changelog deleted file mode 100755 index 3b9efc21fcc..00000000000 --- a/workhorse/_support/changelog +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env ruby -# -# Generate a changelog entry file in the correct location. -# -# Automatically stages the file and amends the previous commit if the `--amend` -# argument is used. -# -# Stolen from gitlab-org/gitaly, lifted from gitlab-org/gitlab-ce - -require 'optparse' -require 'yaml' - -Options = Struct.new( - :amend, - :author, - :dry_run, - :force, - :merge_request, - :title, - :type -) -INVALID_TYPE = -1 - -class ChangelogOptionParser - Type = Struct.new(:name, :description) - TYPES = [ - Type.new('added', 'New feature'), - Type.new('fixed', 'Bug fix'), - Type.new('changed', 'Feature change'), - Type.new('deprecated', 'New deprecation'), - Type.new('removed', 'Feature removal'), - Type.new('security', 'Security fix'), - Type.new('performance', 'Performance improvement'), - Type.new('other', 'Other') - ].freeze - TYPES_OFFSET = 1 - - class << self - def parse(argv) - options = Options.new - - parser = OptionParser.new do |opts| - opts.banner = "Usage: #{__FILE__} [options] [title]\n\n" - - # Note: We do not provide a shorthand for this in order to match the `git - # commit` interface - opts.on('--amend', 'Amend the previous commit') do |value| - options.amend = value - end - - opts.on('-f', '--force', 'Overwrite an existing entry') do |value| - options.force = value - end - - opts.on('-m', '--merge-request [integer]', Integer, 'Merge request ID') do |value| - options.merge_request = value - end - - opts.on('-n', '--dry-run', "Don't actually write anything, just print") do |value| - options.dry_run = value - end - - opts.on('-u', '--git-username', 'Use Git user.name configuration as the author') do |value| - options.author = git_user_name if value - end - - opts.on('-t', '--type [string]', String, "The category of the change, valid options are: #{TYPES.map(&:name).join(', ')}") do |value| - options.type = parse_type(value) - end - - opts.on('-h', '--help', 'Print help message') do - $stdout.puts opts - exit - end - end - - parser.parse!(argv) - - # Title is everything that remains, but let's clean it up a bit - options.title = argv.join(' ').strip.squeeze(' ').tr("\r\n", '') - - options - end - - def read_type - read_type_message - - type = TYPES[$stdin.getc.to_i - TYPES_OFFSET] - assert_valid_type!(type) - - type.name - end - - private - - def parse_type(name) - type_found = TYPES.find do |type| - type.name == name - end - type_found ? type_found.name : INVALID_TYPE - end - - def read_type_message - $stdout.puts "\n>> Please specify the index for the category of your change:" - TYPES.each_with_index do |type, index| - $stdout.puts "#{index + TYPES_OFFSET}. #{type.description}" - end - $stdout.print "\n?> " - end - - def assert_valid_type!(type) - unless type - $stderr.puts "Invalid category index, please select an index between 1 and #{TYPES.length}" - exit 1 - end - end - - def git_user_name - %x{git config user.name}.strip - end - end -end - -class ChangelogEntry - attr_reader :options - - def initialize(options) - @options = options - - assert_feature_branch! - assert_title! - assert_new_file! - - # Read type from $stdin unless is already set - options.type ||= ChangelogOptionParser.read_type - assert_valid_type! - - $stdout.puts "\e[32mcreate\e[0m #{file_path}" - $stdout.puts contents - - unless options.dry_run - write - amend_commit if options.amend - end - end - - private - - def contents - yaml_content = YAML.dump( - 'title' => title, - 'merge_request' => options.merge_request, - 'author' => options.author, - 'type' => options.type - ) - remove_trailing_whitespace(yaml_content) - end - - def write - File.write(file_path, contents) - end - - def amend_commit - %x{git add #{file_path}} - exec("git commit --amend") - end - - def fail_with(message) - $stderr.puts "\e[31merror\e[0m #{message}" - exit 1 - end - - def assert_feature_branch! - return unless branch_name == 'master' - - fail_with "Create a branch first!" - end - - def assert_new_file! - return unless File.exist?(file_path) - return if options.force - - fail_with "#{file_path} already exists! Use `--force` to overwrite." - end - - def assert_title! - return if options.title.length > 0 || options.amend - - fail_with "Provide a title for the changelog entry or use `--amend`" \ - " to use the title from the previous commit." - end - - def assert_valid_type! - return unless options.type && options.type == INVALID_TYPE - - fail_with 'Invalid category given!' - end - - def title - if options.title.empty? - last_commit_subject - else - options.title - end - end - - def last_commit_subject - %x{git log --format="%s" -1}.strip - end - - def file_path - File.join( - unreleased_path, - branch_name.gsub(/[^\w-]/, '-') << '.yml' - ) - end - - def unreleased_path - path = File.join('changelogs', 'unreleased') - path = File.join('ee', path) if ee? - - path - end - - def ee? - @ee ||= File.exist?(File.expand_path('../CHANGELOG-EE.md', __dir__)) - end - - def branch_name - @branch_name ||= %x{git symbolic-ref --short HEAD}.strip - end - - def remove_trailing_whitespace(yaml_content) - yaml_content.gsub(/ +$/, '') - end -end - -if $0 == __FILE__ - options = ChangelogOptionParser.parse(ARGV) - ChangelogEntry.new(options) -end - -# vim: ft=ruby diff --git a/workhorse/_support/check_changelog.sh b/workhorse/_support/check_changelog.sh deleted file mode 100755 index 2f9850f09fd..00000000000 --- a/workhorse/_support/check_changelog.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -set -e - -# we skip the changelog check if the merge requet title ends with "NO CHANGELOG" -if echo "$CI_MERGE_REQUEST_TITLE" | grep -q ' NO CHANGELOG$'; then - echo "Changelog not needed" - - exit 0 -fi - -target=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master} - -if git diff --name-only "origin/$target" | grep -q '^changelogs/' ; then - echo "Changelog included" -else - echo "Please add a changelog running '_support/changelog'" - echo "or disable this check adding 'NO CHANGELOG' at the end of the merge request title" - echo "/title $CI_MERGE_REQUEST_TITLE NO CHANGELOG" - - exit 1 -fi diff --git a/workhorse/_support/generate_changelog b/workhorse/_support/generate_changelog deleted file mode 100755 index a9a8bae5a25..00000000000 --- a/workhorse/_support/generate_changelog +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env ruby -# Generates the changelog from the yaml entries in changelogs/unreleased -# -# Lifted form gitlab-org/gitaly - -require 'yaml' -require 'fileutils' - -class ChangelogEntry - attr_reader :title, :merge_request, :type, :author - - def initialize(file_path) - yaml = YAML.safe_load(File.read(file_path)) - - @title = yaml['title'] - @merge_request = yaml['merge_request'] - @type = yaml['type'] - @author = yaml['author'] - end - - def to_s - str = "" - str << "- #{title}\n" - str << " https://gitlab.com/gitlab-org/gitlab-workhorse/-/merge_requests/#{merge_request}\n" - str << " Contributed by #{author}\n" if author - - str - end -end - -ROOT_DIR = File.expand_path('../..', __FILE__) -UNRELEASED_ENTRIES = File.join(ROOT_DIR, 'changelogs', 'unreleased') -CHANGELOG_FILE = File.join(ROOT_DIR, 'CHANGELOG') - -def main(version) - entries = [] - Dir["#{UNRELEASED_ENTRIES}/*.yml"].each do |yml| - entries << ChangelogEntry.new(yml) - FileUtils.rm(yml) - end - - sections = [] - types = entries.map(&:type).uniq.sort - types.each do |type| - text = '' - text << "### #{type.capitalize}\n" - - entries.each do |e| - next unless e.type == type - - text << e.to_s - end - - sections << text - end - - sections << '- No changes.' if sections.empty? - - new_version_entry = ["## v#{version}\n\n", sections.join("\n"), "\n"].join - - current_changelog = File.read(CHANGELOG_FILE).lines - header = current_changelog.shift(2) - - new_changelog = [header, new_version_entry, current_changelog.join] - - File.write(CHANGELOG_FILE, new_changelog.join) -end - -unless ARGV.count == 1 - warn "Usage: #{$0} VERSION" - warn "Specify version as x.y.z" - abort -end - -main(ARGV.first) diff --git a/workhorse/_support/tag.sh b/workhorse/_support/tag.sh deleted file mode 100644 index 639fd141dad..00000000000 --- a/workhorse/_support/tag.sh +++ /dev/null @@ -1,45 +0,0 @@ -set -e - -main() { - version=$1 - set_version - - changelog - - git commit VERSION -m "Update VERSION to $version" - - tag_name="v${version}" - git tag $TAG_OPTS -m "Version ${version}" -a ${tag_name} - git show ${tag_name} - cat <<'EOF' - - Remember to now push your tag, either to gitlab.com (for a - normal release) or dev.gitlab.org (for a security release). -EOF -} - -set_version() { - if ! echo "${version}" | grep -q '^[0-9]\+\.[0-9]\+\.[0-9]\+$' ; then - echo "Invalid VERSION: ${version}" - exit 1 - fi - - if git tag --list | grep -q "^v${version}$" ; then - echo "Tag already exists for ${version}" - exit 1 - fi - - echo "$version" > VERSION -} - -changelog() { - _support/generate_changelog "$version" - - git commit CHANGELOG changelogs/unreleased --file - <