Use parallel to run tests in parallel

2020-07-30 14:44:08 +02:00 · 2020-07-30 14:44:08 +02:00 · 7abb5c5da1
parent b488ba1e26
commit 7abb5c5da1
7 changed files with 57 additions and 114 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -42,7 +42,7 @@ jobs:

      - name: Setup system dependencies
        run: |
-          sudo apt-get install postgresql-client-common redis-tools
+          sudo apt-get install postgresql-client-common redis-tools parallel
          wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.0.3-64bit-static.tar.xz"
          tar xf ffmpeg-release-4.0.3-64bit-static.tar.xz
          mkdir -p $HOME/bin
--- a/.travis.yml
+++ b/.travis.yml
@ -1,58 +0,0 @@
-language: node_js
-
-node_js:
-  - "10"
-
-git:
-  depth: 1
-
-addons:
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - g++-4.9
-  postgresql: "9.6"
-
-cache:
-  directories:
-    - $HOME/.cache/yarn
-    - $HOME/fixtures
-
-sudo: false
-
-services:
-  - postgresql
-  - redis-server
-
-install:
-  - CC=gcc-4.9 CXX=g++-4.9 yarn install
-
-before_script:
-  - wget --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.0.3-64bit-static.tar.xz"
-  - tar xf ffmpeg-release-4.0.3-64bit-static.tar.xz
-  - mkdir -p $HOME/bin
-  - cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin
-  - export PATH=$HOME/bin:$PATH
-  - psql -c "create user peertube with password 'peertube';" -U postgres
-
-matrix:
-  include:
-  - env: TEST_SUITE=misc
-  - env: TEST_SUITE=api-1
-  - env: TEST_SUITE=api-2
-  - env: TEST_SUITE=api-3
-  - env: TEST_SUITE=api-4
-  - env: TEST_SUITE=cli
-  - env: TEST_SUITE=lint
-
-script:
-  - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE"
-
-after_failure:
-  - cat test1/logs/peertube.log
-  - cat test2/logs/peertube.log
-  - cat test3/logs/peertube.log
-  - cat test4/logs/peertube.log
-  - cat test5/logs/peertube.log
-  - cat test6/logs/peertube.log
--- a/scripts/ci.sh
+++ b/scripts/ci.sh
@ -9,42 +9,82 @@ fi

 killall -q peertube || true

-perl -0777 -i  -pe 's#proxy:(\n\s+)enabled: false\n\s+url: ""#proxy:$1enabled: true$1url: "http://188.165.225.149:7899"#' config/test.yaml
+retries=3
+jobs=2
+
+runTest () {
+    retries=3
+    jobs=$1
+    shift
+    files=$@
+
+    echo $files
+
+    MOCHA_PARALLEL=true parallel -t -j $jobs --retries $retries \
+        npm run mocha -- -c --timeout 30000 --exit --require ts-node/register --require tsconfig-paths/register --bail \
+        ::: $files
+}
+
+findTestFiles () {
+    find $1 -type f -name "*.ts" | grep -v index.ts | xargs echo
+}

 if [ "$1" = "misc" ]; then
    npm run build -- --light
-    TS_NODE_FILES=true mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail \
-        server/tests/client.ts \
-        server/tests/feeds/index.ts \
-        server/tests/misc-endpoints.ts \
-        server/tests/helpers/index.ts \
-        server/tests/plugins/index.ts
+
+    feedsFiles=$(findTestFiles server/tests/feeds)
+    helperFiles=$(findTestFiles server/tests/helpers)
+    pluginsFiles=$(findTestFiles server/tests/plugins)
+    miscFiles=server/tests/client.ts server/tests/misc-endpoints.ts
+
+    TS_NODE_FILES=true runTest 2 $feedsFiles $helperFiles $pluginsFiles $miscFiles
 elif [ "$1" = "cli" ]; then
    npm run build:server
    npm run setup:cli
-    mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail server/tests/cli/index.ts
+
+    cliFiles=$(findTestFiles server/tests/cli)
+
+    runTest 1 $cliFiles
 elif [ "$1" = "api-1" ]; then
    npm run build:server
-    sh ./server/tests/api/ci-1.sh 2
+
+    checkParamFiles=$(findTestFiles server/tests/api/check-params)
+    notificationsFiles=$(findTestFiles server/tests/api/notifications)
+    searchFiles=$(findTestFiles server/tests/api/search)
+
+    runTest 2 $notificationsFiles $searchFiles $checkParamFiles
 elif [ "$1" = "api-2" ]; then
    npm run build:server
-    sh ./server/tests/api/ci-2.sh 2
+
+    serverFiles=$(findTestFiles server/tests/api/server)
+    usersFiles=$(findTestFiles server/tests/api/users)
+
+    runTest 2 $serverFiles $usersFiles
 elif [ "$1" = "api-3" ]; then
    npm run build:server
-    sh ./server/tests/api/ci-3.sh 2
+
+    videosFiles=$(findTestFiles server/tests/api/videos)
+
+    runTest 1 $videosFiles
 elif [ "$1" = "api-4" ]; then
    npm run build:server
-    sh ./server/tests/api/ci-4.sh 2
+
+    activitypubFiles=$(findTestFiles server/tests/api/moderation)
+    redundancyFiles=$(findTestFiles server/tests/api/redundancy)
+    activitypubFiles=$(findTestFiles server/tests/api/activitypub)
+
+    TS_NODE_FILES=true runTest 2 $activitypubFiles $redundancyFiles $activitypubFiles
 elif [ "$1" = "external-plugins" ]; then
    npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail server/tests/external-plugins/index.ts
+
+    externalPluginsFiles=$(findTestFiles server/tests/external-plugins)
+
+    runTest 1 $externalPluginsFiles
 elif [ "$1" = "lint" ]; then
-    npm run eslint -- --ext .ts "server/**/*.ts" "shared/**/*.ts"
+    npm run eslint -- --ext .ts "server/**/*.ts" "shared/**/*.ts" "scripts/**/*.ts"
    npm run swagger-cli -- validate support/doc/api/openapi.yaml

    ( cd client
      npm run lint
    )
 fi
-
-git checkout -- config/test.yaml
--- a/server/tests/api/ci-1.sh
+++ b/server/tests/api/ci-1.sh
@ -1,10 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts | xargs echo)
-notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo)
-searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo)
-
-MOCHA_PARALLEL=true npm run mocha -- --timeout 30000 --exit --require ts-node/register --require tsconfig-paths/register --bail \
-    $notificationsFiles $searchFiles $checkParamFiles
--- a/server/tests/api/ci-2.sh
+++ b/server/tests/api/ci-2.sh
@ -1,10 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo)
-usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo)
-
-MOCHA_PARALLEL=true npm run mocha -- --parallel --jobs $1 --timeout 30000 --exit \
-  --require ts-node/register --require tsconfig-paths/register --bail \
-  $serverFiles $usersFiles
--- a/server/tests/api/ci-3.sh
+++ b/server/tests/api/ci-3.sh
@ -1,8 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo)
-
-npm run mocha -- --timeout 30000 --exit --require ts-node/register --require tsconfig-paths/register --bail \
-    $videosFiles
--- a/server/tests/api/ci-4.sh
+++ b/server/tests/api/ci-4.sh
@ -1,11 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-activitypubFiles=$(find server/tests/api/moderation -type f | grep -v index.ts | xargs echo)
-redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo)
-activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo)
-
-TS_NODE_FILES=true MOCHA_PARALLEL=true npm run mocha -- --parallel --jobs $1 --timeout 30000 --exit \
-  --require ts-node/register --require tsconfig-paths/register --bail \
-  $redundancyFiles $activitypubFiles