gitlab-org--gitlab-foss/spec/services/ci/update_build_state_service_spec.rb

# frozen_string_literal: true

require 'spec_helper'

RSpec.describe Ci::UpdateBuildStateService do
  let_it_be(:project) { create(:project) }
  let_it_be(:pipeline) { create(:ci_pipeline, project: project) }

  let(:build) { create(:ci_build, :running, pipeline: pipeline) }
  let(:metrics) { spy('metrics') }

  subject { described_class.new(build, params) }

  before do
    stub_feature_flags(ci_enable_live_trace: true)
  end

  context 'when build has unknown failure reason' do
    let(:params) do
      {
        output: { checksum: 'crc32:12345678', bytesize: 123 },
        state: 'failed',
        failure_reason: 'no idea here',
        exit_code: 42
      }
    end

    it 'updates a build status' do
      result = subject.execute

      expect(build).to be_failed
      expect(result.status).to eq 200
    end
  end

  context 'when build has failed' do
    let(:params) do
      {
        output: { checksum: 'crc32:12345678', bytesize: 123 },
        state: 'failed',
        failure_reason: 'script_failure',
        exit_code: 7
      }
    end

    it 'sends a build failed event to Snowplow' do
      expect(::Ci::TrackFailedBuildWorker)
        .to receive(:perform_async).with(build.id, params[:exit_code], params[:failure_reason])

      subject.execute
    end
  end

  context 'when build does not have checksum' do
    context 'when state has changed' do
      let(:params) { { state: 'success' } }

      it 'updates a state of a running build' do
        subject.execute

        expect(build).to be_success
      end

      it 'returns 200 OK status' do
        result = subject.execute

        expect(result.status).to eq 200
      end

      it 'does not increment finalized trace metric' do
        execute_with_stubbed_metrics!

        expect(metrics)
          .not_to have_received(:increment_trace_operation)
          .with(operation: :finalized)
      end
    end

    context 'when it is a heartbeat request' do
      let(:params) { { state: 'success' } }

      it 'updates a build timestamp' do
        expect { subject.execute }.to change { build.updated_at }
      end
    end

    context 'when state is unknown' do
      let(:params) { { state: 'unknown' } }

      it 'responds with 400 bad request' do
        result = subject.execute

        expect(result.status).to eq 400
        expect(build).to be_running
      end
    end
  end

  context 'when build has a checksum' do
    let(:params) do
      {
        output: { checksum: 'crc32:12345678', bytesize: 123 },
        state: 'failed',
        failure_reason: 'script_failure',
        exit_code: 42
      }
    end

    context 'when build does not have associated trace chunks' do
      it 'updates a build status' do
        result = subject.execute

        expect(build).to be_failed
        expect(result.status).to eq 200
      end

      it 'updates the allow_failure flag' do
        expect(build)
          .to receive(:drop_with_exit_code!)
          .with('script_failure', 42)
          .and_call_original

        subject.execute
      end

      it 'does not increment invalid trace metric' do
        execute_with_stubbed_metrics!

        expect(metrics)
          .not_to have_received(:increment_trace_operation)
          .with(operation: :invalid)
      end

      it 'does not increment chunks_invalid_checksum trace metric' do
        execute_with_stubbed_metrics!

        expect(metrics)
          .not_to have_received(:increment_error_counter)
          .with(error_reason: :chunks_invalid_checksum)
      end
    end

    context 'when build trace has been migrated' do
      before do
        create(:ci_build_trace_chunk, :persisted, build: build, initial_data: 'abcd')
      end

      it 'updates a build state' do
        subject.execute

        expect(build).to be_failed
      end

      it 'updates the allow_failure flag' do
        expect(build)
          .to receive(:drop_with_exit_code!)
          .with('script_failure', 42)
          .and_call_original

        subject.execute
      end

      it 'responds with 200 OK status' do
        result = subject.execute

        expect(result.status).to eq 200
      end

      it 'does not set a backoff value' do
        result = subject.execute

        expect(result.backoff).to be_nil
      end

      it 'increments trace finalized operation metric' do
        execute_with_stubbed_metrics!

        expect(metrics)
          .to have_received(:increment_trace_operation)
          .with(operation: :finalized)
      end

      it 'records migration duration in a histogram' do
        freeze_time do
          create(:ci_build_pending_state, build: build, created_at: 0.5.seconds.ago)

          execute_with_stubbed_metrics!
        end

        expect(metrics)
          .to have_received(:observe_migration_duration)
          .with(0.5)
      end

      context 'when trace checksum is not valid' do
        it 'increments invalid trace metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .to have_received(:increment_trace_operation)
            .with(operation: :invalid)
        end

        it 'increments chunks_invalid_checksum trace metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_checksum)
        end
      end

      context 'when trace checksum is valid' do
        let(:params) do
          { output: { checksum: 'crc32:ed82cd11', bytesize: 4 }, state: 'success' }
        end

        it 'does not increment invalid or corrupted trace metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .not_to have_received(:increment_trace_operation)
            .with(operation: :invalid)

          expect(metrics)
            .not_to have_received(:increment_trace_operation)
            .with(operation: :corrupted)

          expect(metrics)
            .not_to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_checksum)

          expect(metrics)
            .not_to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_size)
        end

        context 'when using deprecated parameters' do
          let(:params) do
            { checksum: 'crc32:ed82cd11', state: 'success' }
          end

          it 'does not increment invalid or corrupted trace metric' do
            execute_with_stubbed_metrics!

            expect(metrics)
              .not_to have_received(:increment_trace_operation)
              .with(operation: :invalid)

            expect(metrics)
              .not_to have_received(:increment_trace_operation)
              .with(operation: :corrupted)

            expect(metrics)
              .not_to have_received(:increment_error_counter)
              .with(error_reason: :chunks_invalid_checksum)

            expect(metrics)
              .not_to have_received(:increment_error_counter)
              .with(error_reason: :chunks_invalid_size)
          end
        end
      end

      context 'when trace checksum is invalid and the log is corrupted' do
        let(:params) do
          { output: { checksum: 'crc32:12345678', bytesize: 1 }, state: 'success' }
        end

        it 'increments invalid and corrupted trace metrics' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .to have_received(:increment_trace_operation)
            .with(operation: :invalid)

          expect(metrics)
            .to have_received(:increment_trace_operation)
            .with(operation: :corrupted)

          expect(metrics)
            .to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_checksum)

          expect(metrics)
            .to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_size)
        end
      end

      context 'when trace checksum is invalid but the log seems fine' do
        let(:params) do
          { output: { checksum: 'crc32:12345678', bytesize: 4 }, state: 'success' }
        end

        it 'does not increment corrupted trace metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .to have_received(:increment_trace_operation)
            .with(operation: :invalid)

          expect(metrics)
            .to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_checksum)

          expect(metrics)
            .not_to have_received(:increment_trace_operation)
            .with(operation: :corrupted)

          expect(metrics)
            .not_to have_received(:increment_error_counter)
            .with(error_reason: :chunks_invalid_size)
        end
      end

      context 'when failed to acquire a build trace lock' do
        it 'accepts a state update request' do
          build.trace.lock do
            result = subject.execute

            expect(result.status).to eq 202
          end
        end

        it 'increment locked trace metric' do
          build.trace.lock do
            execute_with_stubbed_metrics!

            expect(metrics)
              .to have_received(:increment_trace_operation)
              .with(operation: :locked)
          end
        end
      end
    end

    context 'when build trace has not been migrated yet' do
      before do
        create(:ci_build_trace_chunk, :redis_with_data, build: build)
      end

      it 'does not update a build state' do
        subject.execute

        expect(build).to be_running
      end

      it 'responds with 202 accepted' do
        result = subject.execute

        expect(result.status).to eq 202
      end

      it 'sets a request backoff value' do
        result = subject.execute

        expect(result.backoff.to_i).to be > 0
      end

      it 'schedules live chunks for migration' do
        expect(Ci::BuildTraceChunkFlushWorker)
          .to receive(:perform_async)
          .with(build.trace_chunks.first.id)

        subject.execute
      end

      it 'creates a pending state record' do
        subject.execute

        build.pending_state.then do |status|
          expect(status).to be_present
          expect(status.state).to eq 'failed'
          expect(status.trace_checksum).to eq 'crc32:12345678'
          expect(status.failure_reason).to eq 'script_failure'
        end
      end

      it 'increments trace accepted operation metric' do
        execute_with_stubbed_metrics!

        expect(metrics)
          .to have_received(:increment_trace_operation)
          .with(operation: :accepted)
      end

      it 'does not increment invalid trace metric' do
        execute_with_stubbed_metrics!

        expect(metrics)
          .not_to have_received(:increment_trace_operation)
          .with(operation: :invalid)

        expect(metrics)
          .not_to have_received(:increment_error_counter)
          .with(error_reason: :chunks_invalid_checksum)
      end

      context 'when build pending state is outdated' do
        before do
          build.create_pending_state(
            state: 'failed',
            trace_checksum: 'crc32:12345678',
            failure_reason: 'script_failure',
            created_at: 10.minutes.ago
          )
        end

        it 'responds with 200 OK' do
          result = subject.execute

          expect(result.status).to eq 200
        end

        it 'updates build state' do
          subject.execute

          expect(build.reload).to be_failed
          expect(build.failure_reason).to eq 'script_failure'
        end

        it 'increments discarded traces metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .to have_received(:increment_trace_operation)
            .with(operation: :discarded)
        end

        it 'does not increment finalized trace metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .not_to have_received(:increment_trace_operation)
            .with(operation: :finalized)
        end
      end

      context 'when build pending state has changes' do
        before do
          build.create_pending_state(
            state: 'success',
            created_at: 10.minutes.ago
          )
        end

        it 'uses stored state and responds with 200 OK' do
          result = subject.execute

          expect(result.status).to eq 200
        end

        it 'increments conflict trace metric' do
          execute_with_stubbed_metrics!

          expect(metrics)
            .to have_received(:increment_trace_operation)
            .with(operation: :conflict)
        end
      end

      context 'when live traces are disabled' do
        before do
          stub_feature_flags(ci_enable_live_trace: false)
        end

        it 'responds with 200 OK' do
          result = subject.execute

          expect(result.status).to eq 200
        end
      end
    end
  end

  def execute_with_stubbed_metrics!
    described_class
      .new(build, params, metrics)
      .execute
  end
end