diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 9bdeb4d9b00..8df530c546d 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -4,6 +4,8 @@ - vendor/ruby/ - .yarn-cache/ - tmp/cache/assets/sprockets + - tmp/cache/babel-loader + - tmp/cache/vue-loader .gitlab:assets:compile-metadata: extends: @@ -28,7 +30,7 @@ DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 cache: - key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" + key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:tmp_cache_webpack:v7" artifacts: name: webpack-report expire_in: 31d @@ -84,7 +86,7 @@ gitlab:assets:compile pull-cache: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 cache: - key: "assets-compile:v7" + key: "assets-compile:v8" artifacts: expire_in: 7d paths: @@ -106,7 +108,7 @@ compile-assets pull-push-cache foss: - master cache: policy: pull-push - key: "assets-compile:v7:foss" + key: "assets-compile:v8:foss" compile-assets pull-cache: extends: .compile-assets-metadata @@ -117,7 +119,7 @@ compile-assets pull-cache foss: extends: [".compile-assets-metadata", ".only-ee-as-if-foss"] cache: policy: pull - key: "assets-compile:v7:foss" + key: "assets-compile:v8:foss" .only-code-frontend-job-base: extends: diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/cluster_form_dropdown.vue b/app/assets/javascripts/create_cluster/components/cluster_form_dropdown.vue similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/components/cluster_form_dropdown.vue rename to app/assets/javascripts/create_cluster/components/cluster_form_dropdown.vue diff --git a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue index 4996ce3d342..d187730030d 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue +++ b/app/assets/javascripts/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue @@ -3,7 +3,7 @@ import { createNamespacedHelpers, mapState, mapActions } from 'vuex'; import _ from 'underscore'; import { GlFormInput, GlFormCheckbox } from '@gitlab/ui'; import { sprintf, s__ } from '~/locale'; -import ClusterFormDropdown from './cluster_form_dropdown.vue'; +import ClusterFormDropdown from '~/create_cluster/components/cluster_form_dropdown.vue'; import { KUBERNETES_VERSIONS } from '../constants'; import LoadingButton from '~/vue_shared/components/loading_button.vue'; diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/index.js b/app/assets/javascripts/create_cluster/eks_cluster/store/index.js index 09fd560240d..8dc55506dc2 100644 --- a/app/assets/javascripts/create_cluster/eks_cluster/store/index.js +++ b/app/assets/javascripts/create_cluster/eks_cluster/store/index.js @@ -4,7 +4,7 @@ import * as getters from './getters'; import mutations from './mutations'; import state from './state'; -import clusterDropdownStore from './cluster_dropdown'; +import clusterDropdownStore from '~/create_cluster/store/cluster_dropdown'; import { fetchRoles, diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js b/app/assets/javascripts/create_cluster/store/cluster_dropdown/actions.js similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/actions.js rename to app/assets/javascripts/create_cluster/store/cluster_dropdown/actions.js diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/getters.js b/app/assets/javascripts/create_cluster/store/cluster_dropdown/getters.js similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/getters.js rename to app/assets/javascripts/create_cluster/store/cluster_dropdown/getters.js diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/index.js b/app/assets/javascripts/create_cluster/store/cluster_dropdown/index.js similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/index.js rename to app/assets/javascripts/create_cluster/store/cluster_dropdown/index.js diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/mutation_types.js b/app/assets/javascripts/create_cluster/store/cluster_dropdown/mutation_types.js similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/mutation_types.js rename to app/assets/javascripts/create_cluster/store/cluster_dropdown/mutation_types.js diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/mutations.js b/app/assets/javascripts/create_cluster/store/cluster_dropdown/mutations.js similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/mutations.js rename to app/assets/javascripts/create_cluster/store/cluster_dropdown/mutations.js diff --git a/app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/state.js b/app/assets/javascripts/create_cluster/store/cluster_dropdown/state.js similarity index 100% rename from app/assets/javascripts/create_cluster/eks_cluster/store/cluster_dropdown/state.js rename to app/assets/javascripts/create_cluster/store/cluster_dropdown/state.js diff --git a/changelogs/unreleased/remove-feature-flag-import-graceful-failures.yml b/changelogs/unreleased/remove-feature-flag-import-graceful-failures.yml new file mode 100644 index 00000000000..df6d4429e42 --- /dev/null +++ b/changelogs/unreleased/remove-feature-flag-import-graceful-failures.yml @@ -0,0 +1,5 @@ +--- +title: Remove feature flag for import graceful failures +merge_request: +author: +type: other diff --git a/changelogs/unreleased/sh-fix-sidekiq-timestamps.yml b/changelogs/unreleased/sh-fix-sidekiq-timestamps.yml new file mode 100644 index 00000000000..7da759c4bca --- /dev/null +++ b/changelogs/unreleased/sh-fix-sidekiq-timestamps.yml @@ -0,0 +1,5 @@ +--- +title: Make Sidekiq timestamps consistently ISO 8601 +merge_request: 22750 +author: +type: fixed diff --git a/doc/user/packages/npm_registry/index.md b/doc/user/packages/npm_registry/index.md index 1ac1dcc1897..6b65aaba2c5 100644 --- a/doc/user/packages/npm_registry/index.md +++ b/doc/user/packages/npm_registry/index.md @@ -195,7 +195,7 @@ info Visit https://yarnpkg.com/en/docs/cli/install for documentation about this ``` In this case, try adding this to your `.npmrc` file (and replace `` -with your with your OAuth or personal access token): +with your OAuth or personal access token): ```text //gitlab.com/api/v4/projects/:_authToken= diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index d9c253788b4..886087bc73a 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -75,9 +75,6 @@ module Gitlab save_id_mapping(relation_key, data_hash, relation_object) rescue => e - # re-raise if not enabled - raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true) - log_import_failure(relation_key, relation_index, e) end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index 88888c5994e..e0b0d684bea 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -3,6 +3,8 @@ module Gitlab module SidekiqLogging class JSONFormatter + TIMESTAMP_FIELDS = %w[created_at enqueued_at started_at retried_at failed_at completed_at].freeze + def call(severity, timestamp, progname, data) output = { severity: severity, @@ -13,11 +15,27 @@ module Gitlab when String output[:message] = data when Hash + convert_to_iso8601!(data) output.merge!(data) end output.to_json + "\n" end + + private + + def convert_to_iso8601!(payload) + TIMESTAMP_FIELDS.each do |key| + value = payload[key] + payload[key] = format_time(value) if value.present? + end + end + + def format_time(timestamp) + return timestamp unless timestamp.is_a?(Numeric) + + Time.at(timestamp).utc.iso8601(3) + end end end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 2baa16abe27..8e7626b8eb6 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -6,8 +6,6 @@ require 'active_record/log_subscriber' module Gitlab module SidekiqLogging class StructuredLogger - START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze - DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze MAXIMUM_JOB_ARGUMENTS_LENGTH = 10.kilobytes def call(job, queue) @@ -65,8 +63,6 @@ module Gitlab payload['job_status'] = 'done' end - convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS) - payload['db_duration'] = ActiveRecord::LogSubscriber.runtime payload['db_duration_s'] = payload['db_duration'] / 1000 @@ -79,7 +75,7 @@ module Gitlab # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0 - payload['completed_at'] = Time.now.utc + payload['completed_at'] = Time.now.utc.to_f end def parse_job(job) @@ -91,17 +87,9 @@ module Gitlab job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] job['args'] = limited_job_args(job['args']) if job['args'] - convert_to_iso8601(job, START_TIMESTAMP_FIELDS) - job end - def convert_to_iso8601(payload, keys) - keys.each do |key| - payload[key] = format_time(payload[key]) if payload[key] - end - end - def elapsed(t0) t1 = get_time { @@ -121,12 +109,6 @@ module Gitlab Gitlab::Metrics::System.monotonic_time end - def format_time(timestamp) - return timestamp if timestamp.is_a?(String) - - Time.at(timestamp).utc.iso8601(3) - end - def limited_job_args(args) return unless args.is_a?(Array) diff --git a/spec/frontend/create_cluster/eks_cluster/components/cluster_form_dropdown_spec.js b/spec/frontend/create_cluster/components/cluster_form_dropdown_spec.js similarity index 98% rename from spec/frontend/create_cluster/eks_cluster/components/cluster_form_dropdown_spec.js rename to spec/frontend/create_cluster/components/cluster_form_dropdown_spec.js index 76b262a5196..292b8694fbc 100644 --- a/spec/frontend/create_cluster/eks_cluster/components/cluster_form_dropdown_spec.js +++ b/spec/frontend/create_cluster/components/cluster_form_dropdown_spec.js @@ -2,7 +2,7 @@ import { shallowMount } from '@vue/test-utils'; import $ from 'jquery'; import { GlIcon } from '@gitlab/ui'; -import ClusterFormDropdown from '~/create_cluster/eks_cluster/components/cluster_form_dropdown.vue'; +import ClusterFormDropdown from '~/create_cluster/components/cluster_form_dropdown.vue'; import DropdownButton from '~/vue_shared/components/dropdown/dropdown_button.vue'; import DropdownSearchInput from '~/vue_shared/components/dropdown/dropdown_search_input.vue'; diff --git a/spec/frontend/create_cluster/eks_cluster/components/eks_cluster_configuration_form_spec.js b/spec/frontend/create_cluster/eks_cluster/components/eks_cluster_configuration_form_spec.js index 6c069de8af8..25034dcf5ad 100644 --- a/spec/frontend/create_cluster/eks_cluster/components/eks_cluster_configuration_form_spec.js +++ b/spec/frontend/create_cluster/eks_cluster/components/eks_cluster_configuration_form_spec.js @@ -5,7 +5,7 @@ import { GlFormCheckbox } from '@gitlab/ui'; import EksClusterConfigurationForm from '~/create_cluster/eks_cluster/components/eks_cluster_configuration_form.vue'; import eksClusterFormState from '~/create_cluster/eks_cluster/store/state'; -import clusterDropdownStoreState from '~/create_cluster/eks_cluster/store/cluster_dropdown/state'; +import clusterDropdownStoreState from '~/create_cluster/store/cluster_dropdown/state'; const localVue = createLocalVue(); localVue.use(Vuex); diff --git a/spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/actions_spec.js b/spec/frontend/create_cluster/store/cluster_dropdown/actions_spec.js similarity index 89% rename from spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/actions_spec.js rename to spec/frontend/create_cluster/store/cluster_dropdown/actions_spec.js index 58f8855a64c..014b527161f 100644 --- a/spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/actions_spec.js +++ b/spec/frontend/create_cluster/store/cluster_dropdown/actions_spec.js @@ -1,8 +1,8 @@ import testAction from 'helpers/vuex_action_helper'; -import createState from '~/create_cluster/eks_cluster/store/cluster_dropdown/state'; -import * as types from '~/create_cluster/eks_cluster/store/cluster_dropdown/mutation_types'; -import actionsFactory from '~/create_cluster/eks_cluster/store/cluster_dropdown/actions'; +import createState from '~/create_cluster/store/cluster_dropdown/state'; +import * as types from '~/create_cluster/store/cluster_dropdown/mutation_types'; +import actionsFactory from '~/create_cluster/store/cluster_dropdown/actions'; describe('Cluster dropdown Store Actions', () => { const items = [{ name: 'item 1' }]; diff --git a/spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/mutations_spec.js b/spec/frontend/create_cluster/store/cluster_dropdown/mutations_spec.js similarity index 84% rename from spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/mutations_spec.js rename to spec/frontend/create_cluster/store/cluster_dropdown/mutations_spec.js index 0665047edea..5edd237133d 100644 --- a/spec/frontend/create_cluster/eks_cluster/store/cluster_dropdown/mutations_spec.js +++ b/spec/frontend/create_cluster/store/cluster_dropdown/mutations_spec.js @@ -2,9 +2,9 @@ import { REQUEST_ITEMS, RECEIVE_ITEMS_SUCCESS, RECEIVE_ITEMS_ERROR, -} from '~/create_cluster/eks_cluster/store/cluster_dropdown/mutation_types'; -import createState from '~/create_cluster/eks_cluster/store/cluster_dropdown/state'; -import mutations from '~/create_cluster/eks_cluster/store/cluster_dropdown/mutations'; +} from '~/create_cluster/store/cluster_dropdown/mutation_types'; +import createState from '~/create_cluster/store/cluster_dropdown/state'; +import mutations from '~/create_cluster/store/cluster_dropdown/mutations'; describe('Cluster dropdown store mutations', () => { let state; diff --git a/spec/lib/gitlab/sidekiq_logging/json_formatter_spec.rb b/spec/lib/gitlab/sidekiq_logging/json_formatter_spec.rb index a2cb38ec5b1..f2092334117 100644 --- a/spec/lib/gitlab/sidekiq_logging/json_formatter_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/json_formatter_spec.rb @@ -3,28 +3,53 @@ require 'spec_helper' describe Gitlab::SidekiqLogging::JSONFormatter do - let(:hash_input) { { foo: 1, bar: 'test' } } let(:message) { 'This is a test' } - let(:timestamp) { Time.now } + let(:now) { Time.now } + let(:timestamp) { now.utc.to_f } + let(:timestamp_iso8601) { now.iso8601(3) } - it 'wraps a Hash' do - result = subject.call('INFO', timestamp, 'my program', hash_input) + describe 'with a Hash' do + let(:hash_input) do + { + foo: 1, + 'bar' => 'test', + 'created_at' => timestamp, + 'enqueued_at' => timestamp, + 'started_at' => timestamp, + 'retried_at' => timestamp, + 'failed_at' => timestamp, + 'completed_at' => timestamp_iso8601 + } + end - data = JSON.parse(result) - expected_output = hash_input.stringify_keys - expected_output['severity'] = 'INFO' - expected_output['time'] = timestamp.utc.iso8601(3) + it 'properly formats timestamps into ISO 8601 form' do + result = subject.call('INFO', now, 'my program', hash_input) - expect(data).to eq(expected_output) + data = JSON.parse(result) + expected_output = hash_input.stringify_keys.merge!( + { + 'severity' => 'INFO', + 'time' => timestamp_iso8601, + 'created_at' => timestamp_iso8601, + 'enqueued_at' => timestamp_iso8601, + 'started_at' => timestamp_iso8601, + 'retried_at' => timestamp_iso8601, + 'failed_at' => timestamp_iso8601, + 'completed_at' => timestamp_iso8601 + } + ) + + expect(data).to eq(expected_output) + end end it 'wraps a String' do - result = subject.call('DEBUG', timestamp, 'my string', message) + result = subject.call('DEBUG', now, 'my string', message) data = JSON.parse(result) expected_output = { severity: 'DEBUG', - time: timestamp.utc.iso8601(3), + time: timestamp_iso8601, message: message } diff --git a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb index 20adfe7091c..43cdb998091 100644 --- a/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb +++ b/spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb @@ -30,8 +30,8 @@ describe Gitlab::SidekiqLogging::StructuredLogger do 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: start', 'job_status' => 'start', 'pid' => Process.pid, - 'created_at' => created_at.iso8601(3), - 'enqueued_at' => created_at.iso8601(3), + 'created_at' => created_at.to_f, + 'enqueued_at' => created_at.to_f, 'scheduling_latency_s' => scheduling_latency_s ) end @@ -40,7 +40,7 @@ describe Gitlab::SidekiqLogging::StructuredLogger do 'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec', 'job_status' => 'done', 'duration' => 0.0, - 'completed_at' => timestamp.iso8601(3), + 'completed_at' => timestamp.to_f, 'cpu_s' => 1.111112, 'db_duration' => 0, 'db_duration_s' => 0 @@ -227,17 +227,17 @@ describe Gitlab::SidekiqLogging::StructuredLogger do describe '#add_time_keys!' do let(:time) { { duration: 0.1231234, cputime: 1.2342345 } } let(:payload) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status' } } - let(:current_utc_time) { '2019-09-23 10:00:58 UTC' } - let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time } } + let(:current_utc_time) { Time.now.utc } + let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time.to_f } } subject { described_class.new } it 'update payload correctly' do - expect(Time).to receive_message_chain(:now, :utc).and_return(current_utc_time) + Timecop.freeze(current_utc_time) do + subject.send(:add_time_keys!, time, payload) - subject.send(:add_time_keys!, time, payload) - - expect(payload).to eq(payload_with_time_keys) + expect(payload).to eq(payload_with_time_keys) + end end end end diff --git a/spec/tasks/gitlab/import_export/import_rake_spec.rb b/spec/tasks/gitlab/import_export/import_rake_spec.rb index 18b89912b9f..b824ede03b2 100644 --- a/spec/tasks/gitlab/import_export/import_rake_spec.rb +++ b/spec/tasks/gitlab/import_export/import_rake_spec.rb @@ -76,37 +76,13 @@ describe 'gitlab:import_export:import rake task', :sidekiq do let(:not_imported_message) { /Total number of not imported relations: 1/ } let(:error) { /Validation failed: Notes is invalid/ } - context 'when import_graceful_failures feature flag is enabled' do - before do - stub_feature_flags(import_graceful_failures: true) - end + it 'performs project import successfully' do + expect { subject }.to output(not_imported_message).to_stdout + expect { subject }.not_to raise_error - it 'performs project import successfully' do - expect { subject }.to output(not_imported_message).to_stdout - expect { subject }.not_to raise_error - - expect(project.merge_requests).to be_empty - expect(project.import_state.last_error).to be_nil - expect(project.import_state.status).to eq('finished') - end - end - - context 'when import_graceful_failures feature flag is disabled' do - before do - stub_feature_flags(import_graceful_failures: false) - end - - it 'fails project import with an error' do - # Catch exit call, and raise exception instead - expect_any_instance_of(GitlabProjectImport).to receive(:exit) - .with(1).and_raise(SystemExit) - - expect { subject }.to raise_error(SystemExit).and output(error).to_stdout - - expect(project.merge_requests).to be_empty - expect(project.import_state.last_error).to match(error) - expect(project.import_state.status).to eq('failed') - end + expect(project.merge_requests).to be_empty + expect(project.import_state.last_error).to be_nil + expect(project.import_state.status).to eq('finished') end end end