gitlab-org--gitlab-foss/spec/requests/lfs_http_spec.rb

require 'spec_helper'

describe 'Git LFS API and storage' do
  include WorkhorseHelpers

  let(:user) { create(:user) }
  let!(:lfs_object) { create(:lfs_object, :with_file) }

  let(:headers) do
    {
      'Authorization' => authorization,
      'X-Sendfile-Type' => sendfile
    }.compact
  end
  let(:authorization) { }
  let(:sendfile) { }

  let(:sample_oid) { lfs_object.oid }
  let(:sample_size) { lfs_object.size }

  describe 'when lfs is disabled' do
    let(:project) { create(:empty_project) }
    let(:body) do
      {
        'objects' => [
          { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
            'size' => 1575078
          },
          { 'oid' => sample_oid,
            'size' => sample_size
          }
        ],
        'operation' => 'upload'
      }
    end
    let(:authorization) { authorize_user }

    before do
      allow(Gitlab.config.lfs).to receive(:enabled).and_return(false)
      post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects/batch", body, headers
    end

    it 'responds with 501' do
      expect(response).to have_http_status(501)
      expect(json_response).to include('message' => 'Git LFS is not enabled on this GitLab server, contact your admin.')
    end
  end

  context 'project specific LFS settings' do
    let(:project) { create(:empty_project) }
    let(:body) do
      {
        'objects' => [
          { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
            'size' => 1575078
          },
          { 'oid' => sample_oid,
            'size' => sample_size
          }
        ],
        'operation' => 'upload'
      }
    end
    let(:authorization) { authorize_user }

    context 'with LFS disabled globally' do
      before do
        project.team << [user, :master]
        allow(Gitlab.config.lfs).to receive(:enabled).and_return(false)
      end

      describe 'LFS disabled in project' do
        before do
          project.update_attribute(:lfs_enabled, false)
        end

        it 'responds with a 501 message on upload' do
          post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects/batch", body, headers

          expect(response).to have_http_status(501)
        end

        it 'responds with a 501 message on download' do
          get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers

          expect(response).to have_http_status(501)
        end
      end

      describe 'LFS enabled in project' do
        before do
          project.update_attribute(:lfs_enabled, true)
        end

        it 'responds with a 501 message on upload' do
          post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects/batch", body, headers

          expect(response).to have_http_status(501)
        end

        it 'responds with a 501 message on download' do
          get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers

          expect(response).to have_http_status(501)
        end
      end
    end

    context 'with LFS enabled globally' do
      before do
        project.team << [user, :master]
        enable_lfs
      end

      describe 'LFS disabled in project' do
        before do
          project.update_attribute(:lfs_enabled, false)
        end

        it 'responds with a 403 message on upload' do
          post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects/batch", body, headers

          expect(response).to have_http_status(403)
          expect(json_response).to include('message' => 'Access forbidden. Check your access level.')
        end

        it 'responds with a 403 message on download' do
          get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers

          expect(response).to have_http_status(403)
          expect(json_response).to include('message' => 'Access forbidden. Check your access level.')
        end
      end

      describe 'LFS enabled in project' do
        before do
          project.update_attribute(:lfs_enabled, true)
        end

        it 'responds with a 200 message on upload' do
          post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects/batch", body, headers

          expect(response).to have_http_status(200)
          expect(json_response['objects'].first['size']).to eq(1575078)
        end

        it 'responds with a 200 message on download' do
          get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers

          expect(response).to have_http_status(200)
        end
      end
    end
  end

  describe 'deprecated API' do
    let(:project) { create(:empty_project) }

    before do
      enable_lfs
    end

    shared_examples 'a deprecated' do
      it 'responds with 501' do
        expect(response).to have_http_status(501)
      end

      it 'returns deprecated message' do
        expect(json_response).to include('message' => 'Server supports batch API only, please update your Git LFS client to version 1.0.1 and up.')
      end
    end

    context 'when fetching lfs object using deprecated API' do
      let(:authorization) { authorize_user }

      before do
        get "#{project.http_url_to_repo}/info/lfs/objects/#{sample_oid}", nil, headers
      end

      it_behaves_like 'a deprecated'
    end

    context 'when handling lfs request using deprecated API' do
      let(:authorization) { authorize_user }
      before do
        post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects", nil, headers
      end

      it_behaves_like 'a deprecated'
    end
  end

  describe 'when fetching lfs object' do
    let(:project) { create(:empty_project) }
    let(:update_permissions) { }

    before do
      enable_lfs
      update_permissions
      get "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}", nil, headers
    end

    context 'and request comes from gitlab-workhorse' do
      context 'without user being authorized' do
        it 'responds with status 401' do
          expect(response).to have_http_status(401)
        end
      end

      context 'with required headers' do
        shared_examples 'responds with a file' do
          let(:sendfile) { 'X-Sendfile' }

          it 'responds with status 200' do
            expect(response).to have_http_status(200)
          end

          it 'responds with the file location' do
            expect(response.headers['Content-Type']).to eq('application/octet-stream')
            expect(response.headers['X-Sendfile']).to eq(lfs_object.file.path)
          end
        end

        context 'with user is authorized' do
          let(:authorization) { authorize_user }

          context 'and does not have project access' do
            let(:update_permissions) do
              project.lfs_objects << lfs_object
            end

            it 'responds with status 404' do
              expect(response).to have_http_status(404)
            end
          end

          context 'and does have project access' do
            let(:update_permissions) do
              project.team << [user, :master]
              project.lfs_objects << lfs_object
            end

            it_behaves_like 'responds with a file'
          end
        end

        context 'when CI is authorized' do
          let(:authorization) { authorize_ci_project }

          let(:update_permissions) do
            project.lfs_objects << lfs_object
          end

          it_behaves_like 'responds with a file'
        end
      end

      context 'without required headers' do
        let(:authorization) { authorize_user }

        it 'responds with status 404' do
          expect(response).to have_http_status(404)
        end
      end
    end
  end

  describe 'when handling lfs batch request' do
    let(:update_lfs_permissions) { }
    let(:update_user_permissions) { }

    before do
      enable_lfs
      update_lfs_permissions
      update_user_permissions
      post_lfs_json "#{project.http_url_to_repo}/info/lfs/objects/batch", body, headers
    end

    describe 'download' do
      let(:project) { create(:empty_project) }
      let(:body) do
        { 'operation' => 'download',
          'objects' => [
            { 'oid' => sample_oid,
              'size' => sample_size
            }]
        }
      end

      shared_examples 'an authorized requests' do
        context 'when downloading an lfs object that is assigned to our project' do
          let(:update_lfs_permissions) do
            project.lfs_objects << lfs_object
          end

          it 'responds with status 200' do
            expect(response).to have_http_status(200)
          end

          it 'with href to download' do
            expect(json_response).to eq('objects' => [
              { 'oid' => sample_oid,
                'size' => sample_size,
                'actions' => {
                  'download' => {
                    'href' => "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}",
                    'header' => { 'Authorization' => authorization }
                  }
                }
              }])
          end
        end

        context 'when downloading an lfs object that is assigned to other project' do
          let(:other_project) { create(:empty_project) }
          let(:update_lfs_permissions) do
            other_project.lfs_objects << lfs_object
          end

          it 'responds with status 200' do
            expect(response).to have_http_status(200)
          end

          it 'with href to download' do
            expect(json_response).to eq('objects' => [
              { 'oid' => sample_oid,
                'size' => sample_size,
                'error' => {
                  'code' => 404,
                  'message' => "Object does not exist on the server or you don't have permissions to access it",
                }
              }])
          end
        end

        context 'when downloading a lfs object that does not exist' do
          let(:body) do
            { 'operation' => 'download',
              'objects' => [
                { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
                  'size' => 1575078
                }]
            }
          end

          it 'responds with status 200' do
            expect(response).to have_http_status(200)
          end

          it 'with an 404 for specific object' do
            expect(json_response).to eq('objects' => [
              { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
                'size' => 1575078,
                'error' => {
                  'code' => 404,
                  'message' => "Object does not exist on the server or you don't have permissions to access it",
                }
              }])
          end
        end

        context 'when downloading one new and one existing lfs object' do
          let(:body) do
            { 'operation' => 'download',
              'objects' => [
                { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
                  'size' => 1575078
                },
                { 'oid' => sample_oid,
                  'size' => sample_size
                }
              ]
            }
          end

          let(:update_lfs_permissions) do
            project.lfs_objects << lfs_object
          end

          it 'responds with status 200' do
            expect(response).to have_http_status(200)
          end

          it 'responds with upload hypermedia link for the new object' do
            expect(json_response).to eq('objects' => [
              { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
                'size' => 1575078,
                'error' => {
                  'code' => 404,
                  'message' => "Object does not exist on the server or you don't have permissions to access it",
                }
              },
              { 'oid' => sample_oid,
                'size' => sample_size,
                'actions' => {
                  'download' => {
                    'href' => "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}",
                    'header' => { 'Authorization' => authorization }
                  }
                }
              }])
          end
        end
      end

      context 'when user is authenticated' do
        let(:authorization) { authorize_user }

        let(:update_user_permissions) do
          project.team << [user, role]
        end

        it_behaves_like 'an authorized requests' do
          let(:role) { :reporter }
        end

        context 'when user does is not member of the project' do
          let(:update_user_permissions) { nil }

          it 'responds with 404' do
            expect(response).to have_http_status(404)
          end
        end

        context 'when user does not have download access' do
          let(:role) { :guest }

          it 'responds with 403' do
            expect(response).to have_http_status(403)
          end
        end
      end

      context 'when CI is authorized' do
        let(:authorization) { authorize_ci_project }

        it_behaves_like 'an authorized requests'
      end

      context 'when user is not authenticated' do
        describe 'is accessing public project' do
          let(:project) { create(:project, :public) }

          let(:update_lfs_permissions) do
            project.lfs_objects << lfs_object
          end

          it 'responds with status 200 and href to download' do
            expect(response).to have_http_status(200)
          end

          it 'responds with status 200 and href to download' do
            expect(json_response).to eq('objects' => [
              { 'oid' => sample_oid,
                'size' => sample_size,
                'actions' => {
                  'download' => {
                    'href' => "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}",
                    'header' => {}
                  }
                }
              }])
          end
        end

        describe 'is accessing non-public project' do
          let(:update_lfs_permissions) do
            project.lfs_objects << lfs_object
          end

          it 'responds with authorization required' do
            expect(response).to have_http_status(401)
          end
        end
      end
    end

    describe 'upload' do
      let(:project) { create(:project, :public) }
      let(:body) do
        { 'operation' => 'upload',
          'objects' => [
            { 'oid' => sample_oid,
              'size' => sample_size
            }]
        }
      end

      describe 'when request is authenticated' do
        describe 'when user has project push access' do
          let(:authorization) { authorize_user }

          let(:update_user_permissions) do
            project.team << [user, :developer]
          end

          context 'when pushing an lfs object that already exists' do
            let(:other_project) { create(:empty_project) }
            let(:update_lfs_permissions) do
              other_project.lfs_objects << lfs_object
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'responds with links the object to the project' do
              expect(json_response['objects']).to be_kind_of(Array)
              expect(json_response['objects'].first['oid']).to eq(sample_oid)
              expect(json_response['objects'].first['size']).to eq(sample_size)
              expect(lfs_object.projects.pluck(:id)).not_to include(project.id)
              expect(lfs_object.projects.pluck(:id)).to include(other_project.id)
              expect(json_response['objects'].first['actions']['upload']['href']).to eq("#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}")
              expect(json_response['objects'].first['actions']['upload']['header']).to eq('Authorization' => authorization)
            end
          end

          context 'when pushing a lfs object that does not exist' do
            let(:body) do
              { 'operation' => 'upload',
                'objects' => [
                  { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
                    'size' => 1575078
                  }]
              }
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'responds with upload hypermedia link' do
              expect(json_response['objects']).to be_kind_of(Array)
              expect(json_response['objects'].first['oid']).to eq("91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
              expect(json_response['objects'].first['size']).to eq(1575078)
              expect(json_response['objects'].first['actions']['upload']['href']).to eq("#{Gitlab.config.gitlab.url}/#{project.path_with_namespace}.git/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897/1575078")
              expect(json_response['objects'].first['actions']['upload']['header']).to eq('Authorization' => authorization)
            end
          end

          context 'when pushing one new and one existing lfs object' do
            let(:body) do
              { 'operation' => 'upload',
                'objects' => [
                  { 'oid' => '91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897',
                    'size' => 1575078
                  },
                  { 'oid' => sample_oid,
                    'size' => sample_size
                  }
                ]
              }
            end

            let(:update_lfs_permissions) do
              project.lfs_objects << lfs_object
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'responds with upload hypermedia link for the new object' do
              expect(json_response['objects']).to be_kind_of(Array)

              expect(json_response['objects'].first['oid']).to eq("91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897")
              expect(json_response['objects'].first['size']).to eq(1575078)
              expect(json_response['objects'].first['actions']['upload']['href']).to eq("#{project.http_url_to_repo}/gitlab-lfs/objects/91eff75a492a3ed0dfcb544d7f31326bc4014c8551849c192fd1e48d4dd2c897/1575078")
              expect(json_response['objects'].first['actions']['upload']['header']).to eq("Authorization" => authorization)

              expect(json_response['objects'].last['oid']).to eq(sample_oid)
              expect(json_response['objects'].last['size']).to eq(sample_size)
              expect(json_response['objects'].last).not_to have_key('actions')
            end
          end
        end

        context 'when user does not have push access' do
          let(:authorization) { authorize_user }

          it 'responds with 403' do
            expect(response).to have_http_status(403)
          end
        end

        context 'when CI is authorized' do
          let(:authorization) { authorize_ci_project }

          it 'responds with 401' do
            expect(response).to have_http_status(401)
          end
        end
      end

      context 'when user is not authenticated' do
        context 'when user has push access' do
          let(:update_user_permissions) do
            project.team << [user, :master]
          end

          it 'responds with status 401' do
            expect(response).to have_http_status(401)
          end
        end

        context 'when user does not have push access' do
          it 'responds with status 401' do
            expect(response).to have_http_status(401)
          end
        end
      end

      context 'when CI is authorized' do
        let(:authorization) { authorize_ci_project }

        it 'responds with status 403' do
          expect(response).to have_http_status(401)
        end
      end
    end

    describe 'unsupported' do
      let(:project) { create(:empty_project) }
      let(:authorization) { authorize_user }
      let(:body) do
        { 'operation' => 'other',
          'objects' => [
            { 'oid' => sample_oid,
              'size' => sample_size
            }]
        }
      end

      it 'responds with status 404' do
        expect(response).to have_http_status(404)
      end
    end
  end

  describe 'when pushing a lfs object' do
    before do
      enable_lfs
    end

    shared_examples 'unauthorized' do
      context 'and request is sent by gitlab-workhorse to authorize the request' do
        before do
          put_authorize
        end

        it 'responds with status 401' do
          expect(response).to have_http_status(401)
        end
      end

      context 'and request is sent by gitlab-workhorse to finalize the upload' do
        before do
          put_finalize
        end

        it 'responds with status 401' do
          expect(response).to have_http_status(401)
        end
      end

      context 'and request is sent with a malformed headers' do
        before do
          put_finalize('/etc/passwd')
        end

        it 'does not recognize it as a valid lfs command' do
          expect(response).to have_http_status(401)
        end
      end
    end

    shared_examples 'forbidden' do
      context 'and request is sent by gitlab-workhorse to authorize the request' do
        before do
          put_authorize
        end

        it 'responds with 403' do
          expect(response).to have_http_status(403)
        end
      end

      context 'and request is sent by gitlab-workhorse to finalize the upload' do
        before do
          put_finalize
        end

        it 'responds with 403' do
          expect(response).to have_http_status(403)
        end
      end

      context 'and request is sent with a malformed headers' do
        before do
          put_finalize('/etc/passwd')
        end

        it 'does not recognize it as a valid lfs command' do
          expect(response).to have_http_status(403)
        end
      end
    end

    describe 'to one project' do
      let(:project) { create(:empty_project) }

      describe 'when user is authenticated' do
        let(:authorization) { authorize_user }

        describe 'when user has push access to the project' do
          before do
            project.team << [user, :developer]
          end

          context 'and the request bypassed workhorse' do
            it 'raises an exception' do
              expect { put_authorize(verified: false) }.to raise_error JWT::DecodeError
            end
          end

          context 'and request is sent by gitlab-workhorse to authorize the request' do
            before do
              put_authorize
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'uses the gitlab-workhorse content type' do
              expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
            end

            it 'responds with status 200, location of lfs store and object details' do
              expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload")
              expect(json_response['LfsOid']).to eq(sample_oid)
              expect(json_response['LfsSize']).to eq(sample_size)
            end
          end

          context 'and request is sent by gitlab-workhorse to finalize the upload' do
            before do
              put_finalize
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'lfs object is linked to the project' do
              expect(lfs_object.projects.pluck(:id)).to include(project.id)
            end
          end

          context 'invalid tempfiles' do
            it 'rejects slashes in the tempfile name (path traversal' do
              put_finalize('foo/bar')
              expect(response).to have_http_status(403)
            end

            it 'rejects tempfile names that do not start with the oid' do
              put_finalize("foo#{sample_oid}")
              expect(response).to have_http_status(403)
            end
          end
        end

        describe 'and user does not have push access' do
          before do
            project.team << [user, :reporter]
          end

          it_behaves_like 'forbidden'
        end
      end

      context 'when CI is authenticated' do
        let(:authorization) { authorize_ci_project }

        it_behaves_like 'unauthorized'
      end

      context 'for unauthenticated' do
        it_behaves_like 'unauthorized'
      end
    end

    describe 'to a forked project' do
      let(:upstream_project) { create(:project, :public) }
      let(:project_owner) { create(:user) }
      let(:project) { fork_project(upstream_project, project_owner) }

      describe 'when user is authenticated' do
        let(:authorization) { authorize_user }

        describe 'when user has push access to the project' do
          before do
            project.team << [user, :developer]
          end

          context 'and request is sent by gitlab-workhorse to authorize the request' do
            before do
              put_authorize
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'with location of lfs store and object details' do
              expect(json_response['StoreLFSPath']).to eq("#{Gitlab.config.shared.path}/lfs-objects/tmp/upload")
              expect(json_response['LfsOid']).to eq(sample_oid)
              expect(json_response['LfsSize']).to eq(sample_size)
            end
          end

          context 'and request is sent by gitlab-workhorse to finalize the upload' do
            before do
              put_finalize
            end

            it 'responds with status 200' do
              expect(response).to have_http_status(200)
            end

            it 'lfs object is linked to the source project' do
              expect(lfs_object.projects.pluck(:id)).to include(upstream_project.id)
            end
          end
        end

        describe 'and user does not have push access' do
          it_behaves_like 'forbidden'
        end
      end

      context 'when CI is authenticated' do
        let(:authorization) { authorize_ci_project }

        it_behaves_like 'unauthorized'
      end

      context 'for unauthenticated' do
        it_behaves_like 'unauthorized'
      end

      describe 'and second project not related to fork or a source project' do
        let(:second_project) { create(:empty_project) }
        let(:authorization) { authorize_user }

        before do
          second_project.team << [user, :master]
          upstream_project.lfs_objects << lfs_object
        end

        context 'when pushing the same lfs object to the second project' do
          before do
            put "#{second_project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil,
                headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp_file).compact
          end

          it 'responds with status 200' do
            expect(response).to have_http_status(200)
          end

          it 'links the lfs object to the project' do
            expect(lfs_object.projects.pluck(:id)).to include(second_project.id, upstream_project.id)
          end
        end
      end
    end

    def put_authorize(verified: true)
      authorize_headers = headers
      authorize_headers.merge!(workhorse_internal_api_request_header) if verified

      put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}/authorize", nil, authorize_headers
    end

    def put_finalize(lfs_tmp = lfs_tmp_file)
      put "#{project.http_url_to_repo}/gitlab-lfs/objects/#{sample_oid}/#{sample_size}", nil,
          headers.merge('X-Gitlab-Lfs-Tmp' => lfs_tmp).compact
    end

    def lfs_tmp_file
      "#{sample_oid}012345678"
    end
  end

  def enable_lfs
    allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
  end

  def authorize_ci_project
    ActionController::HttpAuthentication::Basic.encode_credentials('gitlab-ci-token', project.runners_token)
  end

  def authorize_user
    ActionController::HttpAuthentication::Basic.encode_credentials(user.username, user.password)
  end

  def fork_project(project, user, object = nil)
    allow(RepositoryForkWorker).to receive(:perform_async).and_return(true)
    Projects::ForkService.new(project, user, {}).execute
  end

  def post_lfs_json(url, body = nil, headers = nil)
    post(url, body.try(:to_json), (headers || {}).merge('Content-Type' => 'application/vnd.git-lfs+json'))
  end

  def json_response
    @json_response ||= JSON.parse(response.body)
  end
end