image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.6-golang-1.9-git-2.14-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6" .dedicated-runner: &dedicated-runner retry: 1 tags: - gitlab-org .default-cache: &default-cache key: "ruby-235-with-yarn" paths: - vendor/ruby - .yarn-cache/ .push-cache: &push-cache cache: <<: *default-cache policy: push .pull-cache: &pull-cache cache: <<: *default-cache policy: pull variables: MYSQL_ALLOW_EMPTY_PASSWORD: "1" RAILS_ENV: "test" NODE_ENV: "test" SIMPLECOV: "true" GIT_DEPTH: "20" GIT_SUBMODULE_STRATEGY: "none" GET_SOURCES_ATTEMPTS: "3" KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json before_script: - bundle --version - source scripts/utils.sh - source scripts/prepare_build.sh stages: - build - prepare - test - post-test - pages - post-cleanup # Predefined scopes .tests-metadata-state: &tests-metadata-state <<: *dedicated-runner variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: - source scripts/utils.sh artifacts: expire_in: 31d paths: - knapsack/ - rspec_flaky/ .use-pg: &use-pg services: - postgres:9.2 - redis:alpine .use-mysql: &use-mysql services: - mysql:latest - redis:alpine # Skip all jobs except the ones that begin with 'docs/'. # Used for commits including ONLY documentation changes. # https://docs.gitlab.com/ce/development/writing_documentation.html#testing .except-docs: &except-docs except: - /(^docs[\/-].*|.*-docs$)/ .except-qa: &except-qa except: - /(^qa[\/-].*|.*-qa$)/ .except-docs-and-qa: &except-docs-and-qa except: - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ .rspec-metadata: &rspec-metadata <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test script: - JOB_NAME=( $CI_JOB_NAME ) - export CI_NODE_INDEX=${JOB_NAME[-2]} - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH} - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${JOB_NAME[0]}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${JOB_NAME[0]}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export FLAKY_RSPEC_GENERATE_REPORT=true - export CACHE_CLASSES=true - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}' - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}' - scripts/gitaly-test-spawn - knapsack rspec "--color --format documentation" artifacts: expire_in: 31d when: always paths: - coverage/ - knapsack/ - rspec_flaky/ - tmp/capybara/ .rspec-metadata-pg: &rspec-metadata-pg <<: *rspec-metadata <<: *use-pg .rspec-metadata-mysql: &rspec-metadata-mysql <<: *rspec-metadata <<: *use-mysql .spinach-metadata: &spinach-metadata <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test script: - JOB_NAME=( $CI_JOB_NAME ) - export CI_NODE_INDEX=${JOB_NAME[-2]} - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - export CACHE_CLASSES=true - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - scripts/gitaly-test-spawn - knapsack spinach "-r rerun" -b || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -b -r rerun $(cat tmp/spinach-rerun.txt)' artifacts: expire_in: 31d when: always paths: - coverage/ - knapsack/ - tmp/capybara/ .spinach-metadata-pg: &spinach-metadata-pg <<: *spinach-metadata <<: *use-pg .spinach-metadata-mysql: &spinach-metadata-mysql <<: *spinach-metadata <<: *use-mysql .only-canonical-masters: &only-canonical-masters only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - master@gitlab/gitlabhq - master@gitlab/gitlab-ee ## # Trigger a package build in omnibus-gitlab repository # package-qa: <<: *dedicated-runner image: ruby:2.4-alpine before_script: [] stage: build cache: {} when: manual script: - scripts/trigger-build-omnibus only: - //@gitlab-org/gitlab-ce - //@gitlab-org/gitlab-ee # Review docs base .review-docs: &review-docs <<: *dedicated-runner <<: *except-qa image: ruby:2.4-alpine before_script: - gem install gitlab --no-doc # We need to download the script rather than clone the repo since the # review-docs-cleanup job will not be able to run when the branch gets # deleted (when merging the MR). - apk add --update openssl - wget https://gitlab.com/gitlab-org/gitlab-ce/raw/master/scripts/trigger-build-docs - chmod 755 trigger-build-docs cache: {} dependencies: [] variables: GIT_STRATEGY: none when: manual only: - branches # Trigger a docs build in gitlab-docs # Useful to preview the docs changes live review-docs-deploy: <<: *review-docs stage: build environment: name: review-docs/$CI_COMMIT_REF_NAME # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are secret variables # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 url: http://$DOCS_GITLAB_REPO_SUFFIX-$CI_COMMIT_REF_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX on_stop: review-docs-cleanup script: - ./trigger-build-docs deploy # Cleanup remote environment of gitlab-docs review-docs-cleanup: <<: *review-docs stage: post-cleanup environment: name: review-docs/$CI_COMMIT_REF_NAME action: stop script: - ./trigger-build-docs cleanup # Retrieve knapsack and rspec_flaky reports retrieve-tests-metadata: <<: *tests-metadata-state <<: *except-docs-and-qa stage: prepare cache: key: tests_metadata policy: pull script: - mkdir -p knapsack/${CI_PROJECT_NAME}/ - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}' - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}' - mkdir -p rspec_flaky/ - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' update-tests-metadata: <<: *tests-metadata-state <<: *only-canonical-masters stage: post-test cache: key: tests_metadata paths: - knapsack/ - rspec_flaky/ policy: push script: - retry gem install fog-aws mime-types - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH' - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json flaky-examples-check: <<: *dedicated-runner image: ruby:2.3-alpine services: [] before_script: [] variables: SETUP_DB: "false" USE_BUNDLE_INSTALL: "false" NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json stage: post-test allow_failure: true retry: 0 only: - branches except: - master - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ artifacts: expire_in: 30d paths: - rspec_flaky/ script: - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}' - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT setup-test-env: <<: *dedicated-runner <<: *except-docs <<: *use-pg stage: prepare cache: <<: *default-cache script: - node --version - yarn install --frozen-lockfile --cache-folder .yarn-cache - bundle exec rake gitlab:assets:compile - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' - scripts/gitaly-test-build # Do not use 'bundle exec' here artifacts: expire_in: 7d paths: - node_modules - public/assets - tmp/tests rspec-pg 0 26: *rspec-metadata-pg rspec-pg 1 26: *rspec-metadata-pg rspec-pg 2 26: *rspec-metadata-pg rspec-pg 3 26: *rspec-metadata-pg rspec-pg 4 26: *rspec-metadata-pg rspec-pg 5 26: *rspec-metadata-pg rspec-pg 6 26: *rspec-metadata-pg rspec-pg 7 26: *rspec-metadata-pg rspec-pg 8 26: *rspec-metadata-pg rspec-pg 9 26: *rspec-metadata-pg rspec-pg 10 26: *rspec-metadata-pg rspec-pg 11 26: *rspec-metadata-pg rspec-pg 12 26: *rspec-metadata-pg rspec-pg 13 26: *rspec-metadata-pg rspec-pg 14 26: *rspec-metadata-pg rspec-pg 15 26: *rspec-metadata-pg rspec-pg 16 26: *rspec-metadata-pg rspec-pg 17 26: *rspec-metadata-pg rspec-pg 18 26: *rspec-metadata-pg rspec-pg 19 26: *rspec-metadata-pg rspec-pg 20 26: *rspec-metadata-pg rspec-pg 21 26: *rspec-metadata-pg rspec-pg 22 26: *rspec-metadata-pg rspec-pg 23 26: *rspec-metadata-pg rspec-pg 24 26: *rspec-metadata-pg rspec-pg 25 26: *rspec-metadata-pg rspec-mysql 0 26: *rspec-metadata-mysql rspec-mysql 1 26: *rspec-metadata-mysql rspec-mysql 2 26: *rspec-metadata-mysql rspec-mysql 3 26: *rspec-metadata-mysql rspec-mysql 4 26: *rspec-metadata-mysql rspec-mysql 5 26: *rspec-metadata-mysql rspec-mysql 6 26: *rspec-metadata-mysql rspec-mysql 7 26: *rspec-metadata-mysql rspec-mysql 8 26: *rspec-metadata-mysql rspec-mysql 9 26: *rspec-metadata-mysql rspec-mysql 10 26: *rspec-metadata-mysql rspec-mysql 11 26: *rspec-metadata-mysql rspec-mysql 12 26: *rspec-metadata-mysql rspec-mysql 13 26: *rspec-metadata-mysql rspec-mysql 14 26: *rspec-metadata-mysql rspec-mysql 15 26: *rspec-metadata-mysql rspec-mysql 16 26: *rspec-metadata-mysql rspec-mysql 17 26: *rspec-metadata-mysql rspec-mysql 18 26: *rspec-metadata-mysql rspec-mysql 19 26: *rspec-metadata-mysql rspec-mysql 20 26: *rspec-metadata-mysql rspec-mysql 21 26: *rspec-metadata-mysql rspec-mysql 22 26: *rspec-metadata-mysql rspec-mysql 23 26: *rspec-metadata-mysql rspec-mysql 24 26: *rspec-metadata-mysql rspec-mysql 25 26: *rspec-metadata-mysql spinach-pg 0 4: *spinach-metadata-pg spinach-pg 1 4: *spinach-metadata-pg spinach-pg 2 4: *spinach-metadata-pg spinach-pg 3 4: *spinach-metadata-pg spinach-mysql 0 4: *spinach-metadata-mysql spinach-mysql 1 4: *spinach-metadata-mysql spinach-mysql 2 4: *spinach-metadata-mysql spinach-mysql 3 4: *spinach-metadata-mysql # Static analysis jobs .ruby-static-analysis: &ruby-static-analysis <<: *pull-cache variables: SIMPLECOV: "false" SETUP_DB: "false" .rake-exec: &rake-exec <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache <<: *ruby-static-analysis stage: test script: - bundle exec rake $CI_JOB_NAME static-analysis: <<: *dedicated-runner <<: *except-docs <<: *ruby-static-analysis stage: test script: - scripts/static-analysis # Documentation checks: # - Check validity of relative links # - Make sure cURL examples in API docs use the full switches docs lint: <<: *dedicated-runner <<: *except-qa image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine" stage: test cache: {} dependencies: [] before_script: [] script: - scripts/lint-doc.sh - scripts/lint-changelog-yaml - mv doc/ /nanoc/content/ - cd /nanoc # Build HTML from Markdown - bundle exec nanoc # Check the internal links - bundle exec nanoc check internal_links downtime_check: <<: *rake-exec except: - master - tags - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ ee_compat_check: <<: *rake-exec except: - master - tags - /^[\d-]+-stable(-ee)?/ - branches@gitlab-org/gitlab-ee - branches@gitlab/gitlab-ee retry: 0 artifacts: name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" when: always expire_in: 10d paths: - ee_compat_check/patches/*.patch # DB migration, rollback, and seed jobs .db-migrate-reset: &db-migrate-reset <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test script: - bundle exec rake db:migrate:reset db:migrate:reset-pg: <<: *db-migrate-reset <<: *use-pg db:migrate:reset-mysql: <<: *db-migrate-reset <<: *use-mysql db:check-schema-pg: <<: *db-migrate-reset <<: *use-pg script: - source scripts/schema_changed.sh .migration-paths: &migration-paths <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test variables: SETUP_DB: "false" CREATE_DB_USER: "true" script: - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0 - git checkout -f FETCH_HEAD - bundle install $BUNDLE_INSTALL_FLAGS - cp config/gitlab.yml.example config/gitlab.yml - bundle exec rake db:drop db:create db:schema:load db:seed_fu - git checkout $CI_COMMIT_SHA - bundle install $BUNDLE_INSTALL_FLAGS - . scripts/prepare_build.sh - bundle exec rake db:migrate migration:path-pg: <<: *migration-paths <<: *use-pg migration:path-mysql: <<: *migration-paths <<: *use-mysql .db-rollback: &db-rollback <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test script: - bundle exec rake db:rollback STEP=119 - bundle exec rake db:migrate db:rollback-pg: <<: *db-rollback <<: *use-pg db:rollback-mysql: <<: *db-rollback <<: *use-mysql .db-seed_fu: &db-seed_fu <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test variables: SIZE: "1" SETUP_DB: "false" CREATE_DB_USER: "true" script: - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - bundle exec rake db:setup db:seed_fu artifacts: when: on_failure expire_in: 1d paths: - log/development.log db:seed_fu-pg: <<: *db-seed_fu <<: *use-pg db:seed_fu-mysql: <<: *db-seed_fu <<: *use-mysql # Frontend-related jobs gitlab:assets:compile: <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test dependencies: [] variables: NODE_ENV: "production" RAILS_ENV: "production" SETUP_DB: "false" SKIP_STORAGE_VALIDATION: "true" WEBPACK_REPORT: "true" NO_COMPRESSION: "true" script: - yarn install --frozen-lockfile --production --cache-folder .yarn-cache - bundle exec rake gitlab:assets:compile artifacts: name: webpack-report expire_in: 31d paths: - webpack-report/ karma: <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache <<: *use-pg stage: test variables: BABEL_ENV: "coverage" CHROME_LOG_FILE: "chrome_debug.log" script: - scripts/gitaly-test-spawn - bundle exec rake gettext:po_to_json - bundle exec rake karma coverage: '/^Statements *: (\d+\.\d+%)/' artifacts: name: coverage-javascript expire_in: 31d when: always paths: - chrome_debug.log - coverage-javascript/ codequality: <<: *except-docs <<: *pull-cache before_script: [] image: docker:latest stage: test variables: SETUP_DB: "false" DOCKER_DRIVER: overlay services: - docker:dind script: - cp .rubocop.yml .rubocop.yml.bak - grep -v "rubocop-gitlab-security" .rubocop.yml.bak > .rubocop.yml - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc dev.gitlab.org:5005/gitlab/gitlab-build-images:gitlab-codeclimate-v2 analyze -f json > raw_codeclimate.json - cat raw_codeclimate.json | docker run -i stedolan/jq -c 'map({check_name,fingerprint,location})' > codeclimate.json - mv .rubocop.yml.bak .rubocop.yml artifacts: paths: [codeclimate.json] sast: image: registry.gitlab.com/gitlab-org/gl-sast:latest before_script: [] script: - /app/bin/run . artifacts: paths: [gl-sast-report.json] qa:internal: <<: *dedicated-runner <<: *except-docs stage: test variables: SETUP_DB: "false" services: [] script: - cd qa/ - bundle install - bundle exec rspec coverage: <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: post-test services: [] variables: SETUP_DB: "false" USE_BUNDLE_INSTALL: "true" script: - bundle exec scripts/merge-simplecov coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage expire_in: 31d paths: - coverage/index.html - coverage/assets/ lint:javascript:report: <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: post-test dependencies: - setup-test-env before_script: [] script: - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files - yarn run eslint-report || true # ignore exit code artifacts: name: eslint-report expire_in: 31d paths: - eslint-report.html pages: <<: *dedicated-runner <<: *pull-cache before_script: [] stage: pages dependencies: - coverage - karma - gitlab:assets:compile - lint:javascript:report script: - mv public/ .public/ - mkdir public/ - mv coverage/ public/coverage-ruby/ || true - mv coverage-javascript/ public/coverage-javascript/ || true - mv eslint-report.html public/ || true - mv webpack-report/ public/webpack-report/ || true artifacts: paths: - public only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: <<: *dedicated-runner <<: *pull-cache only: - tags variables: SETUP_DB: "false" script: - bundle package --all --all-platforms artifacts: paths: - vendor/cache only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee gitlab_git_test: <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache variables: SETUP_DB: "false" script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes